diff --git a/public_law/dates.py b/public_law/dates.py
index 00a6e1f8..56278770 100644
--- a/public_law/dates.py
+++ b/public_law/dates.py
@@ -1,3 +1,5 @@
+"""Provide date-related functions."""
+
from typing import Protocol
from datetime import datetime, date
@@ -17,6 +19,12 @@ def todays_date() -> str:
return iso_8601(today())
+def current_year() -> int:
+ """Provide the current year."""
+
+ return today().year
+
+
def today() -> date:
"""Provide today's date in the given timezone."""
@@ -24,10 +32,9 @@ def today() -> date:
# But the Scrapy settings don't seem to be
# available in this context.
# See https://doc.scrapy.org/en/latest/topics/settings.html.
- mountain = pytz.timezone(
- "US/Mountain"
- )
- return mountain.localize(datetime.now()).date()
+ tz = pytz.timezone("US/Mountain")
+
+ return tz.localize(datetime.now()).date()
def iso_8601(a_date: date) -> str:
diff --git a/public_law/parsers/aus/ip_glossary.py b/public_law/parsers/aus/ip_glossary.py
index af170ba0..63c7148f 100644
--- a/public_law/parsers/aus/ip_glossary.py
+++ b/public_law/parsers/aus/ip_glossary.py
@@ -60,6 +60,6 @@ def _parse_mod_date(html: HtmlResponse) -> date:
26 March 2021
"""
mod_date_str: str = cast(str, (
- html.selector.css("span.date-display-single").xpath("@content").get() # type: ignore
+ html.selector.css("span.date-display-single").xpath("@content").get()
))
return datetime.fromisoformat(mod_date_str).date()
diff --git a/public_law/parsers/can/doj_glossaries.py b/public_law/parsers/can/doj_glossaries.py
index 3a116e32..5b4e25fd 100644
--- a/public_law/parsers/can/doj_glossaries.py
+++ b/public_law/parsers/can/doj_glossaries.py
@@ -102,9 +102,9 @@
}
-def configured_urls() -> tuple[str, ...]:
+def configured_urls() -> list[str]:
"""All the URLs that have been properly set up with subjects."""
- return tuple(SUBJECTS.keys())
+ return list(SUBJECTS.keys())
def parse_glossary(html: HtmlResponse) -> GlossaryParseResult:
@@ -114,7 +114,7 @@ def parse_glossary(html: HtmlResponse) -> GlossaryParseResult:
entries: list[GlossaryEntry] = []
match html.css("main dl"):
- case [first, *_] if isinstance(first, Selector):
+ case [first, *_]:
first_dl_list = first
case _:
raise ParseException("Expected a
")
diff --git a/public_law/parsers/usa/colorado/crs.py b/public_law/parsers/usa/colorado/crs.py
index 5255a3a1..c30c03b6 100644
--- a/public_law/parsers/usa/colorado/crs.py
+++ b/public_law/parsers/usa/colorado/crs.py
@@ -1,12 +1,7 @@
# pyright: reportUnknownMemberType=false
-# pyright: reportOptionalMemberAccess=false
-# pyright: reportUnknownVariableType=false
-# pyright: reportUnknownArgumentType=false
-# pyright: reportUnknownLambdaType=false
-
from scrapy.selector.unified import Selector
-from scrapy.http.response import Response
+from scrapy.http.response.xml import XmlResponse
from typing import Any
@@ -16,23 +11,29 @@
from public_law.parsers.usa.colorado.crs_divisions import parse_divisions
-def parse_title_bang(dom: Response, logger: Any) -> Title:
+def parse_title_bang(dom: XmlResponse, logger: Any) -> Title:
match parse_title(dom, logger):
case None:
raise Exception("Could not parse title")
- case result:
- return result
+ case title:
+ return title
-def parse_title(dom: Response, logger: Any) -> Title | None:
- raw_name = dom.xpath("//TITLE-TEXT/text()").get()
-
- if raw_name is None:
- logger.warn(f"Could not parse title name in {dom.url}")
- return None
+def parse_title(dom: XmlResponse, logger: Any) -> Title | None:
+ match(dom.xpath("//TITLE-TEXT/text()").get()):
+ case str(raw_name):
+ name = NonemptyString(titleize(raw_name))
+ case None:
+ logger.warn(f"Could not the parse title name in {dom.url}")
+ return None
+
+ match(dom.xpath("//TITLE-NUM/text()").get()):
+ case str(raw_number):
+ number = NonemptyString(raw_number.split(" ")[1])
+ case None:
+ logger.warn(f"Could not the parse title number in {dom.url}")
+ return None
- name = NonemptyString(titleize(raw_name))
- number = NonemptyString(dom.xpath("//TITLE-NUM/text()").get().split(" ")[1])
url_number = number.rjust(2, "0")
source_url = URL(f"https://leg.colorado.gov/sites/default/files/images/olls/crs2022-title-{url_number}.pdf")
@@ -44,13 +45,15 @@ def parse_title(dom: Response, logger: Any) -> Title | None:
)
-def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Division] | list[Article]:
+def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | XmlResponse, logger: Any) -> list[Division] | list[Article]:
division_nodes = dom.xpath("//T-DIV")
article_nodes = dom.xpath("//TA-LIST")
if len(division_nodes) > 0:
- return parse_divisions(title_number, dom, logger)
+ func = parse_divisions
elif len(article_nodes) > 0:
- return parse_articles(title_number, dom, logger)
+ func = parse_articles
else:
- raise Exception(f"Could not parse divisions or articles in Title {title_number}")
+ raise Exception(f"Could not parse divisions or articles in Title {title_number}. Neither T-DIV nor TA-LIST nodes were found.")
+
+ return func(title_number, dom, logger)
diff --git a/public_law/parsers/usa/colorado/crs_articles.py b/public_law/parsers/usa/colorado/crs_articles.py
index fab29429..ac7d18ae 100644
--- a/public_law/parsers/usa/colorado/crs_articles.py
+++ b/public_law/parsers/usa/colorado/crs_articles.py
@@ -1,8 +1,4 @@
# pyright: reportUnknownMemberType=false
-# pyright: reportOptionalMemberAccess=false
-# pyright: reportUnknownVariableType=false
-# pyright: reportUnknownArgumentType=false
-# pyright: reportUnknownLambdaType=false
from itertools import takewhile, dropwhile
from typing import Any
diff --git a/public_law/parsers/usa/colorado/crs_divisions.py b/public_law/parsers/usa/colorado/crs_divisions.py
index 1f4239e8..ec8ba318 100644
--- a/public_law/parsers/usa/colorado/crs_divisions.py
+++ b/public_law/parsers/usa/colorado/crs_divisions.py
@@ -1,8 +1,4 @@
# pyright: reportUnknownMemberType=false
-# pyright: reportOptionalMemberAccess=false
-# pyright: reportUnknownVariableType=false
-# pyright: reportUnknownArgumentType=false
-# pyright: reportUnknownLambdaType=false
from scrapy.selector.unified import Selector
@@ -22,7 +18,7 @@
def parse_divisions(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Division]:
division_nodes = dom.xpath("//T-DIV")
- divs = []
+ divs: list[Division] = []
for div_node in division_nodes:
raw_div_name = div_name_text(div_node)
diff --git a/public_law/parsers/usa/colorado/crs_sections.py b/public_law/parsers/usa/colorado/crs_sections.py
index 9b079f76..723c6c0c 100644
--- a/public_law/parsers/usa/colorado/crs_sections.py
+++ b/public_law/parsers/usa/colorado/crs_sections.py
@@ -1,13 +1,9 @@
# pyright: reportUnknownMemberType=false
-# pyright: reportOptionalMemberAccess=false
-# pyright: reportUnknownVariableType=false
-# pyright: reportUnknownArgumentType=false
-# pyright: reportUnknownLambdaType=false
from typing import Any
from bs4 import BeautifulSoup
-from scrapy.http.response import Response
+from scrapy.http.response.xml import XmlResponse
from scrapy.selector.unified import Selector
from public_law.selector_util import just_text
@@ -16,10 +12,10 @@
-def parse_sections(dom: Response, logger: Any) -> list[Section]:
+def parse_sections(dom: XmlResponse, logger: Any) -> list[Section]:
section_nodes = dom.xpath("//SECTION-TEXT")
- sections = []
+ sections: list[Section] = []
for node in section_nodes:
if _is_repealed(node):
continue
diff --git a/public_law/parsers/usa/georgia_ag_opinions.py b/public_law/parsers/usa/georgia_ag_opinions.py
index 8e0ee117..d85eec67 100644
--- a/public_law/parsers/usa/georgia_ag_opinions.py
+++ b/public_law/parsers/usa/georgia_ag_opinions.py
@@ -48,12 +48,12 @@ def parse_ag_opinion(html: Response) -> OpinionParseResult:
join("\n"),
),
)
- citation_set = pipe(
+ citation_set = cast(CitationSet, pipe(
re.findall(r"\d+-\d+-\d+(?:\([-().A-Za-z0-9]*[-A-Za-z0-9]\))?", full_text),
set,
sorted,
CitationSet,
- )
+ ))
return OpinionParseResult(
summary=summary,
diff --git a/public_law/selector_util.py b/public_law/selector_util.py
index d85bdc62..65783a9a 100644
--- a/public_law/selector_util.py
+++ b/public_law/selector_util.py
@@ -3,8 +3,9 @@
from scrapy.selector.unified import Selector, SelectorList
-def node_name(node: Selector):
+def node_name(node: Selector) -> str | None:
return node.xpath("name()").get()
+
def just_text(node: Selector | SelectorList | Any) -> str | None:
return node.xpath("text()").get()
diff --git a/public_law/spiders/usa/colorado_crs.py b/public_law/spiders/usa/colorado_crs.py
index d3d66044..c960ae82 100644
--- a/public_law/spiders/usa/colorado_crs.py
+++ b/public_law/spiders/usa/colorado_crs.py
@@ -1,21 +1,17 @@
# pyright: reportUnknownMemberType=false
-# pyright: reportUnknownArgumentType=false
# pyright: reportUnknownVariableType=false
-# pyright: reportUnknownParameterType=false
# pyright: reportGeneralTypeIssues=false
-# pyright: reportUnusedCallResult=false
import os
-import re
-
from pathlib import Path
+from typing import Any
from progressbar import ProgressBar
from scrapy import Spider
from scrapy.http.request import Request
from scrapy.http.response.html import HtmlResponse
-from typing import Any
+from public_law import dates
from public_law.parsers.usa.colorado.crs import parse_title
from public_law.parsers.usa.colorado.crs_sections import parse_sections
@@ -25,7 +21,7 @@ class ColoradoCRS(Spider):
Reads the sources from a local directory instead of the web.
"""
- name = "usa_colorado_crs"
+ name = "usa_colorado_crs"
def start_requests(self):
@@ -53,21 +49,11 @@ def start_requests(self):
def parse(self, response: HtmlResponse, **_: dict[str, Any]): # type: ignore[override]
if "README.txt" in response.url:
- yield from self.parse_readme(response)
+ yield { "kind": "CRS", "edition": dates.current_year() }
else:
yield from self.parse_title_xml(response)
- def parse_readme(self, response: HtmlResponse, **_: dict[str, Any]):
- result = re.findall(r'COLORADO REVISED STATUTES (\d\d\d\d) DATASET', str(response.body))
- if len(result) != 1:
- raise Exception(f"Could not parse year from README: {response.body}")
-
- year: str = result[0]
-
- yield { "kind": "CRS", "edition": int(year) }
-
-
def parse_title_xml(self, response: HtmlResponse, **_: dict[str, Any]):
"""Framework callback which parses one XML file."""
self.logger.debug(f"Parsing {response.url}...")
diff --git a/public_law/spiders/usa/georgia_ag_opinions.py b/public_law/spiders/usa/georgia_ag_opinions.py
index 2b4fde10..2f63102e 100644
--- a/public_law/spiders/usa/georgia_ag_opinions.py
+++ b/public_law/spiders/usa/georgia_ag_opinions.py
@@ -6,11 +6,13 @@
# pyright: reportGeneralTypeIssues=false
-from ...parsers.usa.georgia_ag_opinions import parse_ag_opinion
from scrapy import Spider
from scrapy.http.request import Request
from scrapy.http.response import Response
-from typing import Any, Dict, cast
+from scrapy.http.response.html import HtmlResponse
+from typing import Any, Dict
+
+from ...parsers.usa.georgia_ag_opinions import parse_ag_opinion
JD_VERBOSE_NAME = "USA / Georgia"
PUBLICATION_NAME = "Attorney General Opinions"
@@ -29,23 +31,28 @@ class GeorgiaAgOpinions(Spider):
"https://law.georgia.gov/opinions/unofficial",
]
+
def parse(self, response: Response, **kwargs: Dict[str, Any]):
"""Framework callback which begins the parsing."""
- return self.parse_index_page(response)
- def parse_index_page(self, response: Response):
+ match(response):
+ case HtmlResponse():
+ return self.parse_index_page(response)
+
+ case _:
+ raise Exception(f"Unexpected response type: {type(response)}")
+
+
+ def parse_index_page(self, response: HtmlResponse):
#
# 1. Find all the individual opinions on this index page
# and request a parse for each.
#
- opinion_paths = cast(
- list[str],
- response.xpath(
+ opinion_paths = response.xpath(
"//td[contains(@class, 'views-field-title')]/a/@href"
- ).getall(),
- )
+ ).getall()
- for url in [cast(str, response.urljoin(p)) for p in opinion_paths]:
+ for url in [response.urljoin(p) for p in opinion_paths]:
yield Request(url, callback=self.parse_opinion_page)
#
@@ -60,5 +67,6 @@ def parse_index_page(self, response: Response):
response.urljoin(next_page_path), callback=self.parse_index_page
)
- def parse_opinion_page(self, response: Response):
+
+ def parse_opinion_page(self, response: HtmlResponse):
yield parse_ag_opinion(response)._asdict()
diff --git a/public_law/spiders/usa/oregon_regs.py b/public_law/spiders/usa/oregon_regs.py
index a3149b7e..8c22b9fa 100644
--- a/public_law/spiders/usa/oregon_regs.py
+++ b/public_law/spiders/usa/oregon_regs.py
@@ -106,7 +106,7 @@ def from_crawler(cls, crawler: Crawler, *args: List[str], **kwargs: Dict[str, An
"""Override to register to receive the idle event"""
spider = cast(OregonRegs, super(OregonRegs, cls).from_crawler(crawler, *args, **kwargs))
- crawler.signals.connect(spider.spider_idle, signal=scrapy.signals.spider_idle) # type: ignore
+ crawler.signals.connect(spider.spider_idle, signal=scrapy.signals.spider_idle)
return spider
def spider_idle(self, spider: Spider):
diff --git a/pyproject.toml b/pyproject.toml
index af1a2a44..5028fd08 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ typeCheckingMode = "strict"
reportCallInDefaultInitializer = "error"
reportImplicitStringConcatenation = "error"
reportMissingSuperCall = "error"
+reportMissingTypeStubs = false
reportPropertyTypeMismatch = "error"
reportUninitializedInstanceVariable = "error"
reportUnnecessaryTypeIgnoreComment = "error"
diff --git a/script/create-crs-json b/script/create-crs-json
new file mode 100755
index 00000000..6352824c
--- /dev/null
+++ b/script/create-crs-json
@@ -0,0 +1,3 @@
+#!/usr/bin/env fish
+
+scrapy crawl -a crsdata_dir=tmp/sources --overwrite-output tmp/crs.json:jsonl usa_colorado_crs
diff --git a/script/create-crs-xml-files-notes b/script/create-crs-xml-files-notes
index c0110b8e..1518575d 100755
--- a/script/create-crs-xml-files-notes
+++ b/script/create-crs-xml-files-notes
@@ -1,8 +1,5 @@
#!/usr/bin/env fish
-#
-# So far, just a history of commands I ran.
-#
cd tmp/sources/TITLES
diff --git a/typings/scrapy/__init__.pyi b/typings/scrapy/__init__.pyi
deleted file mode 100644
index f8bc8ba8..00000000
--- a/typings/scrapy/__init__.pyi
+++ /dev/null
@@ -1,22 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import pkgutil
-import sys
-import warnings
-from twisted import version as _txv
-from scrapy.http import FormRequest, Request
-from scrapy.item import Field, Item
-from scrapy.selector import Selector
-from scrapy.spiders import Spider
-
-"""
-Scrapy - a web crawling and web scraping framework written for Python
-"""
-__all__ = ["__version__", "version_info", "twisted_version", "Spider", "Request", "FormRequest", "Selector", "Item", "Field"]
-__version__ = ...
-version_info = ...
-twisted_version = ...
-if sys.version_info < (3, 8):
- ...
diff --git a/typings/scrapy/__main__.pyi b/typings/scrapy/__main__.pyi
deleted file mode 100644
index cdba0a06..00000000
--- a/typings/scrapy/__main__.pyi
+++ /dev/null
@@ -1,6 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-if __name__ == "__main__":
- ...
diff --git a/typings/scrapy/addons.pyi b/typings/scrapy/addons.pyi
deleted file mode 100644
index ce44131a..00000000
--- a/typings/scrapy/addons.pyi
+++ /dev/null
@@ -1,30 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import TYPE_CHECKING
-from scrapy.settings import Settings
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-logger = ...
-class AddonManager:
- """This class facilitates loading and storing :ref:`topics-addons`."""
- def __init__(self, crawler: Crawler) -> None:
- ...
-
- def load_settings(self, settings: Settings) -> None:
- """Load add-ons and configurations from a settings object and apply them.
-
- This will load the add-on for every add-on path in the
- ``ADDONS`` setting and execute their ``update_settings`` methods.
-
- :param settings: The :class:`~scrapy.settings.Settings` object from \
- which to read the add-on configuration
- :type settings: :class:`~scrapy.settings.Settings`
- """
- ...
-
-
-
diff --git a/typings/scrapy/cmdline.pyi b/typings/scrapy/cmdline.pyi
deleted file mode 100644
index 8cd98b9a..00000000
--- a/typings/scrapy/cmdline.pyi
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import argparse
-
-class ScrapyArgumentParser(argparse.ArgumentParser):
- ...
-
-
-def execute(argv=..., settings=...):
- ...
-
-if __name__ == "__main__":
- ...
diff --git a/typings/scrapy/commands/__init__.pyi b/typings/scrapy/commands/__init__.pyi
deleted file mode 100644
index eccc4237..00000000
--- a/typings/scrapy/commands/__init__.pyi
+++ /dev/null
@@ -1,97 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import argparse
-import os
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-from twisted.python import failure
-from scrapy.crawler import CrawlerProcess
-from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
-
-"""
-Base class for Scrapy commands
-"""
-class ScrapyCommand:
- requires_project = ...
- crawler_process: Optional[CrawlerProcess] = ...
- default_settings: Dict[str, Any] = ...
- exitcode = ...
- def __init__(self) -> None:
- ...
-
- def set_crawler(self, crawler): # -> None:
- ...
-
- def syntax(self): # -> Literal['']:
- """
- Command syntax (preferably one-line). Do not include command name.
- """
- ...
-
- def short_desc(self): # -> Literal['']:
- """
- A short description of the command
- """
- ...
-
- def long_desc(self): # -> Literal['']:
- """A long description of the command. Return short description when not
- available. It cannot contain newlines since contents will be formatted
- by optparser which removes newlines and wraps text.
- """
- ...
-
- def help(self): # -> Literal['']:
- """An extensive help for the command. It will be shown when using the
- "help" command. It can contain newlines since no post-formatting will
- be applied to its contents.
- """
- ...
-
- def add_options(self, parser): # -> None:
- """
- Populate option parse with options available for this command
- """
- ...
-
- def process_options(self, args, opts): # -> None:
- ...
-
- def run(self, args: List[str], opts: argparse.Namespace) -> None:
- """
- Entry point for running commands
- """
- ...
-
-
-
-class BaseRunSpiderCommand(ScrapyCommand):
- """
- Common class used to share functionality between the crawl, parse and runspider commands
- """
- def add_options(self, parser): # -> None:
- ...
-
- def process_options(self, args, opts): # -> None:
- ...
-
-
-
-class ScrapyHelpFormatter(argparse.HelpFormatter):
- """
- Help Formatter for scrapy command line help messages.
- """
- def __init__(self, prog, indent_increment=..., max_help_position=..., width=...) -> None:
- ...
-
- def format_part_strings(self, part_strings):
- """
- Underline and title case command line help message headers.
- """
- ...
-
-
-
diff --git a/typings/scrapy/contracts/__init__.pyi b/typings/scrapy/contracts/__init__.pyi
deleted file mode 100644
index eea3d100..00000000
--- a/typings/scrapy/contracts/__init__.pyi
+++ /dev/null
@@ -1,51 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import re
-import sys
-from functools import wraps
-from inspect import getmembers
-from types import CoroutineType
-from typing import AsyncGenerator, Dict
-from unittest import TestCase
-from scrapy.http import Request
-from scrapy.utils.python import get_spec
-from scrapy.utils.spider import iterate_spider_output
-
-class Contract:
- """Abstract class for contracts"""
- request_cls = ...
- def __init__(self, method, *args) -> None:
- ...
-
- def add_pre_hook(self, request, results):
- ...
-
- def add_post_hook(self, request, results):
- ...
-
- def adjust_request_args(self, args):
- ...
-
-
-
-class ContractsManager:
- contracts: Dict[str, Contract] = ...
- def __init__(self, contracts) -> None:
- ...
-
- def tested_methods_from_spidercls(self, spidercls): # -> list[Any]:
- ...
-
- def extract_contracts(self, method): # -> list[Any]:
- ...
-
- def from_spider(self, spider, results): # -> list[Any]:
- ...
-
- def from_method(self, method, results): # -> Request | None:
- ...
-
-
-
diff --git a/typings/scrapy/core/__init__.pyi b/typings/scrapy/core/__init__.pyi
deleted file mode 100644
index a0e30e24..00000000
--- a/typings/scrapy/core/__init__.pyi
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Scrapy core library classes and functions.
-"""
diff --git a/typings/scrapy/core/downloader/__init__.pyi b/typings/scrapy/core/downloader/__init__.pyi
deleted file mode 100644
index 1387ada3..00000000
--- a/typings/scrapy/core/downloader/__init__.pyi
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import random
-from collections import deque
-from datetime import datetime
-from time import time
-from typing import Any, Deque, Dict, Set, TYPE_CHECKING, Tuple, cast
-from twisted.internet import task
-from twisted.internet.defer import Deferred
-from scrapy import Request, Spider, signals
-from scrapy.core.downloader.handlers import DownloadHandlers
-from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
-from scrapy.http import Response
-from scrapy.resolver import dnscache
-from scrapy.settings import BaseSettings
-from scrapy.signalmanager import SignalManager
-from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-class Slot:
- """Downloader slot"""
- def __init__(self, concurrency: int, delay: float, randomize_delay: bool) -> None:
- ...
-
- def free_transfer_slots(self) -> int:
- ...
-
- def download_delay(self) -> float:
- ...
-
- def close(self) -> None:
- ...
-
- def __repr__(self) -> str:
- ...
-
- def __str__(self) -> str:
- ...
-
-
-
-class Downloader:
- DOWNLOAD_SLOT = ...
- def __init__(self, crawler: Crawler) -> None:
- ...
-
- def fetch(self, request: Request, spider: Spider) -> Deferred:
- ...
-
- def needs_backout(self) -> bool:
- ...
-
- def close(self) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/core/downloader/handlers/__init__.pyi b/typings/scrapy/core/downloader/handlers/__init__.pyi
deleted file mode 100644
index 4d980648..00000000
--- a/typings/scrapy/core/downloader/handlers/__init__.pyi
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import logging
-from typing import Any, Callable, Dict, Generator, TYPE_CHECKING, Union, cast
-from twisted.internet import defer
-from twisted.internet.defer import Deferred
-from scrapy import Request, Spider, signals
-from scrapy.exceptions import NotConfigured, NotSupported
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.python import without_none_values
-from scrapy.crawler import Crawler
-
-"""Download handlers for different schemes"""
-if TYPE_CHECKING:
- ...
-logger = ...
-class DownloadHandlers:
- def __init__(self, crawler: Crawler) -> None:
- ...
-
- def download_request(self, request: Request, spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/core/downloader/middleware.pyi b/typings/scrapy/core/downloader/middleware.pyi
deleted file mode 100644
index 9bac3668..00000000
--- a/typings/scrapy/core/downloader/middleware.pyi
+++ /dev/null
@@ -1,22 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Callable
-from twisted.internet.defer import Deferred
-from scrapy import Spider
-from scrapy.http import Request
-from scrapy.middleware import MiddlewareManager
-
-"""
-Downloader Middleware manager
-
-See documentation in docs/topics/downloader-middleware.rst
-"""
-class DownloaderMiddlewareManager(MiddlewareManager):
- component_name = ...
- def download(self, download_func: Callable, request: Request, spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/core/engine.pyi b/typings/scrapy/core/engine.pyi
deleted file mode 100644
index 0d627102..00000000
--- a/typings/scrapy/core/engine.pyi
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Callable, Generator, Iterable, TYPE_CHECKING
-from twisted.internet.defer import Deferred, inlineCallbacks
-from scrapy.http import Request
-from scrapy.spiders import Spider
-from scrapy.utils.reactor import CallLaterOnce
-from scrapy.core.scheduler import BaseScheduler
-from scrapy.crawler import Crawler
-
-"""
-This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
-
-For more information see docs/topics/architecture.rst
-
-"""
-if TYPE_CHECKING:
- ...
-logger = ...
-class Slot:
- def __init__(self, start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce, scheduler: BaseScheduler) -> None:
- ...
-
- def add_request(self, request: Request) -> None:
- ...
-
- def remove_request(self, request: Request) -> None:
- ...
-
- def close(self) -> Deferred:
- ...
-
-
-
-class ExecutionEngine:
- def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None:
- ...
-
- @inlineCallbacks
- def start(self) -> Generator[Deferred, Any, None]:
- ...
-
- def stop(self) -> Deferred:
- """Gracefully stop the execution engine"""
- ...
-
- def close(self) -> Deferred:
- """
- Gracefully close the execution engine.
- If it has already been started, stop it. In all cases, close the spider and the downloader.
- """
- ...
-
- def pause(self) -> None:
- ...
-
- def unpause(self) -> None:
- ...
-
- def spider_is_idle(self) -> bool:
- ...
-
- def crawl(self, request: Request) -> None:
- """Inject the request into the spider <-> downloader pipeline"""
- ...
-
- def download(self, request: Request) -> Deferred:
- """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
- ...
-
- @inlineCallbacks
- def open_spider(self, spider: Spider, start_requests: Iterable = ..., close_if_idle: bool = ...) -> Generator[Deferred, Any, None]:
- ...
-
- def close_spider(self, spider: Spider, reason: str = ...) -> Deferred:
- """Close (cancel) spider and clear all its outstanding requests"""
- ...
-
-
-
diff --git a/typings/scrapy/core/scheduler.pyi b/typings/scrapy/core/scheduler.pyi
deleted file mode 100644
index c04f8c12..00000000
--- a/typings/scrapy/core/scheduler.pyi
+++ /dev/null
@@ -1,217 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from abc import abstractmethod
-from typing import Any, Optional, TYPE_CHECKING, Type, TypeVar
-from twisted.internet.defer import Deferred
-from scrapy.crawler import Crawler
-from scrapy.dupefilters import BaseDupeFilter
-from scrapy.http.request import Request
-from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
-from typing_extensions import Self
-
-if TYPE_CHECKING:
- ...
-logger = ...
-class BaseSchedulerMeta(type):
- """
- Metaclass to check scheduler classes against the necessary interface
- """
- def __instancecheck__(cls, instance: Any) -> bool:
- ...
-
- def __subclasscheck__(cls, subclass: type) -> bool:
- ...
-
-
-
-class BaseScheduler(metaclass=BaseSchedulerMeta):
- """
- The scheduler component is responsible for storing requests received from
- the engine, and feeding them back upon request (also to the engine).
-
- The original sources of said requests are:
-
- * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks
- * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods
- * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods
-
- The order in which the scheduler returns its stored requests (via the ``next_request`` method)
- plays a great part in determining the order in which those requests are downloaded.
-
- The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
- """
- @classmethod
- def from_crawler(cls, crawler: Crawler) -> Self:
- """
- Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
- """
- ...
-
- def open(self, spider: Spider) -> Optional[Deferred]:
- """
- Called when the spider is opened by the engine. It receives the spider
- instance as argument and it's useful to execute initialization code.
-
- :param spider: the spider object for the current crawl
- :type spider: :class:`~scrapy.spiders.Spider`
- """
- ...
-
- def close(self, reason: str) -> Optional[Deferred]:
- """
- Called when the spider is closed by the engine. It receives the reason why the crawl
- finished as argument and it's useful to execute cleaning code.
-
- :param reason: a string which describes the reason why the spider was closed
- :type reason: :class:`str`
- """
- ...
-
- @abstractmethod
- def has_pending_requests(self) -> bool:
- """
- ``True`` if the scheduler has enqueued requests, ``False`` otherwise
- """
- ...
-
- @abstractmethod
- def enqueue_request(self, request: Request) -> bool:
- """
- Process a request received by the engine.
-
- Return ``True`` if the request is stored correctly, ``False`` otherwise.
-
- If ``False``, the engine will fire a ``request_dropped`` signal, and
- will not make further attempts to schedule the request at a later time.
- For reference, the default Scrapy scheduler returns ``False`` when the
- request is rejected by the dupefilter.
- """
- ...
-
- @abstractmethod
- def next_request(self) -> Optional[Request]:
- """
- Return the next :class:`~scrapy.http.Request` to be processed, or ``None``
- to indicate that there are no requests to be considered ready at the moment.
-
- Returning ``None`` implies that no request from the scheduler will be sent
- to the downloader in the current reactor cycle. The engine will continue
- calling ``next_request`` until ``has_pending_requests`` is ``False``.
- """
- ...
-
-
-
-SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
-class Scheduler(BaseScheduler):
- """
- Default Scrapy scheduler. This implementation also handles duplication
- filtering via the :setting:`dupefilter `.
-
- This scheduler stores requests into several priority queues (defined by the
- :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues
- are backed by either memory or disk based queues (respectively defined by the
- :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings).
-
- Request prioritization is almost entirely delegated to the priority queue. The only
- prioritization performed by this scheduler is using the disk-based queue if present
- (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based
- queue if a serialization error occurs. If the disk queue is not present, the memory one
- is used directly.
-
- :param dupefilter: An object responsible for checking and filtering duplicate requests.
- The value for the :setting:`DUPEFILTER_CLASS` setting is used by default.
- :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar:
- any class that implements the `BaseDupeFilter` interface
-
- :param jobdir: The path of a directory to be used for persisting the crawl's state.
- The value for the :setting:`JOBDIR` setting is used by default.
- See :ref:`topics-jobs`.
- :type jobdir: :class:`str` or ``None``
-
- :param dqclass: A class to be used as persistent request queue.
- The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
- :type dqclass: class
-
- :param mqclass: A class to be used as non-persistent request queue.
- The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
- :type mqclass: class
-
- :param logunser: A boolean that indicates whether or not unserializable requests should be logged.
- The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
- :type logunser: bool
-
- :param stats: A stats collector object to record stats about the request scheduling process.
- The value for the :setting:`STATS_CLASS` setting is used by default.
- :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar:
- any class that implements the `StatsCollector` interface
-
- :param pqclass: A class to be used as priority queue for requests.
- The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
- :type pqclass: class
-
- :param crawler: The crawler object corresponding to the current crawl.
- :type crawler: :class:`scrapy.crawler.Crawler`
- """
- def __init__(self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = ..., dqclass=..., mqclass=..., logunser: bool = ..., stats: Optional[StatsCollector] = ..., pqclass=..., crawler: Optional[Crawler] = ...) -> None:
- ...
-
- @classmethod
- def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
- """
- Factory method, initializes the scheduler with arguments taken from the crawl settings
- """
- ...
-
- def has_pending_requests(self) -> bool:
- ...
-
- def open(self, spider: Spider) -> Optional[Deferred]:
- """
- (1) initialize the memory queue
- (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
- (3) return the result of the dupefilter's ``open`` method
- """
- ...
-
- def close(self, reason: str) -> Optional[Deferred]:
- """
- (1) dump pending requests to disk if there is a disk queue
- (2) return the result of the dupefilter's ``close`` method
- """
- ...
-
- def enqueue_request(self, request: Request) -> bool:
- """
- Unless the received request is filtered out by the Dupefilter, attempt to push
- it into the disk queue, falling back to pushing it into the memory queue.
-
- Increment the appropriate stats, such as: ``scheduler/enqueued``,
- ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
-
- Return ``True`` if the request was stored successfully, ``False`` otherwise.
- """
- ...
-
- def next_request(self) -> Optional[Request]:
- """
- Return a :class:`~scrapy.http.Request` object from the memory queue,
- falling back to the disk queue if the memory queue is empty.
- Return ``None`` if there are no more enqueued requests.
-
- Increment the appropriate stats, such as: ``scheduler/dequeued``,
- ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
- """
- ...
-
- def __len__(self) -> int:
- """
- Return the total amount of enqueued requests
- """
- ...
-
-
-
diff --git a/typings/scrapy/core/scraper.pyi b/typings/scrapy/core/scraper.pyi
deleted file mode 100644
index 46356f7c..00000000
--- a/typings/scrapy/core/scraper.pyi
+++ /dev/null
@@ -1,71 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, AsyncIterable, Generator, Iterable, TYPE_CHECKING, Tuple, Union
-from twisted.internet.defer import Deferred, inlineCallbacks
-from twisted.python.failure import Failure
-from scrapy import Spider
-from scrapy.http import Request, Response
-from scrapy.crawler import Crawler
-
-"""This module implements the Scraper component which parses responses and
-extracts information from them"""
-if TYPE_CHECKING:
- ...
-QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
-logger = ...
-class Slot:
- """Scraper slot (one per running spider)"""
- MIN_RESPONSE_SIZE = ...
- def __init__(self, max_active_size: int = ...) -> None:
- ...
-
- def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
- ...
-
- def next_response_request_deferred(self) -> QueueTuple:
- ...
-
- def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
- ...
-
- def is_idle(self) -> bool:
- ...
-
- def needs_backout(self) -> bool:
- ...
-
-
-
-class Scraper:
- def __init__(self, crawler: Crawler) -> None:
- ...
-
- @inlineCallbacks
- def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
- """Open the given spider for scraping and allocate resources for it"""
- ...
-
- def close_spider(self, spider: Spider) -> Deferred:
- """Close a spider being scraped and release its resources"""
- ...
-
- def is_idle(self) -> bool:
- """Return True if there isn't any more spiders to process"""
- ...
-
- def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
- ...
-
- def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
- ...
-
- def handle_spider_error(self, _failure: Failure, request: Request, response: Union[Response, Failure], spider: Spider) -> None:
- ...
-
- def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, response: Union[Response, Failure], spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/core/spidermw.pyi b/typings/scrapy/core/spidermw.pyi
deleted file mode 100644
index dfd0ccf0..00000000
--- a/typings/scrapy/core/spidermw.pyi
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Callable, Iterable, Union
-from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
-from scrapy import Request, Spider
-from scrapy.http import Response
-from scrapy.middleware import MiddlewareManager
-
-"""
-Spider Middleware manager
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-logger = ...
-ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
-class SpiderMiddlewareManager(MiddlewareManager):
- component_name = ...
- def __init__(self, *middlewares: Any) -> None:
- ...
-
- def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred:
- ...
-
- def process_start_requests(self, start_requests: Iterable[Request], spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/crawler.pyi b/typings/scrapy/crawler.pyi
deleted file mode 100644
index b6f77ea0..00000000
--- a/typings/scrapy/crawler.pyi
+++ /dev/null
@@ -1,146 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Dict, Generator, TYPE_CHECKING, Type, Union
-from twisted.internet.defer import Deferred, inlineCallbacks
-from scrapy import Spider
-from scrapy.settings import Settings
-
-if TYPE_CHECKING:
- ...
-logger = ...
-class Crawler:
- def __init__(self, spidercls: Type[Spider], settings: Union[None, Dict[str, Any], Settings] = ..., init_reactor: bool = ...) -> None:
- ...
-
- @inlineCallbacks
- def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
- ...
-
- @inlineCallbacks
- def stop(self) -> Generator[Deferred, Any, None]:
- """Starts a graceful stop of the crawler and returns a deferred that is
- fired when the crawler is stopped."""
- ...
-
-
-
-class CrawlerRunner:
- """
- This is a convenient helper class that keeps track of, manages and runs
- crawlers inside an already setup :mod:`~twisted.internet.reactor`.
-
- The CrawlerRunner object must be instantiated with a
- :class:`~scrapy.settings.Settings` object.
-
- This class shouldn't be needed (since Scrapy is responsible of using it
- accordingly) unless writing scripts that manually handle the crawling
- process. See :ref:`run-from-script` for an example.
- """
- crawlers = ...
- def __init__(self, settings: Union[Dict[str, Any], Settings, None] = ...) -> None:
- ...
-
- def crawl(self, crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any) -> Deferred:
- """
- Run a crawler with the provided arguments.
-
- It will call the given Crawler's :meth:`~Crawler.crawl` method, while
- keeping track of it so it can be stopped later.
-
- If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
- instance, this method will try to create one using this parameter as
- the spider class given to it.
-
- Returns a deferred that is fired when the crawling is finished.
-
- :param crawler_or_spidercls: already created crawler, or a spider class
- or spider's name inside the project to create it
- :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
- :class:`~scrapy.spiders.Spider` subclass or string
-
- :param args: arguments to initialize the spider
-
- :param kwargs: keyword arguments to initialize the spider
- """
- ...
-
- def create_crawler(self, crawler_or_spidercls: Union[Type[Spider], str, Crawler]) -> Crawler:
- """
- Return a :class:`~scrapy.crawler.Crawler` object.
-
- * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
- * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
- is constructed for it.
- * If ``crawler_or_spidercls`` is a string, this function finds
- a spider with this name in a Scrapy project (using spider loader),
- then creates a Crawler instance for it.
- """
- ...
-
- def stop(self) -> Deferred:
- """
- Stops simultaneously all the crawling jobs taking place.
-
- Returns a deferred that is fired when they all have ended.
- """
- ...
-
- @inlineCallbacks
- def join(self) -> Generator[Deferred, Any, None]:
- """
- join()
-
- Returns a deferred that is fired when all managed :attr:`crawlers` have
- completed their executions.
- """
- ...
-
-
-
-class CrawlerProcess(CrawlerRunner):
- """
- A class to run multiple scrapy crawlers in a process simultaneously.
-
- This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
- for starting a :mod:`~twisted.internet.reactor` and handling shutdown
- signals, like the keyboard interrupt command Ctrl-C. It also configures
- top-level logging.
-
- This utility should be a better fit than
- :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
- :mod:`~twisted.internet.reactor` within your application.
-
- The CrawlerProcess object must be instantiated with a
- :class:`~scrapy.settings.Settings` object.
-
- :param install_root_handler: whether to install root logging handler
- (default: True)
-
- This class shouldn't be needed (since Scrapy is responsible of using it
- accordingly) unless writing scripts that manually handle the crawling
- process. See :ref:`run-from-script` for an example.
- """
- def __init__(self, settings: Union[Dict[str, Any], Settings, None] = ..., install_root_handler: bool = ...) -> None:
- ...
-
- def start(self, stop_after_crawl: bool = ..., install_signal_handlers: bool = ...) -> None:
- """
- This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
- size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
- based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
-
- If ``stop_after_crawl`` is True, the reactor will be stopped after all
- crawlers have finished, using :meth:`join`.
-
- :param bool stop_after_crawl: stop or not the reactor when all
- crawlers have finished
-
- :param bool install_signal_handlers: whether to install the shutdown
- handlers (default: True)
- """
- ...
-
-
-
diff --git a/typings/scrapy/downloadermiddlewares/__init__.pyi b/typings/scrapy/downloadermiddlewares/__init__.pyi
deleted file mode 100644
index 006bc274..00000000
--- a/typings/scrapy/downloadermiddlewares/__init__.pyi
+++ /dev/null
@@ -1,4 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
diff --git a/typings/scrapy/dupefilters.pyi b/typings/scrapy/dupefilters.pyi
deleted file mode 100644
index 9dbe7f6d..00000000
--- a/typings/scrapy/dupefilters.pyi
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Optional, TYPE_CHECKING
-from twisted.internet.defer import Deferred
-from scrapy.http.request import Request
-from scrapy.settings import BaseSettings
-from scrapy.spiders import Spider
-from scrapy.utils.request import RequestFingerprinterProtocol
-from typing_extensions import Self
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-class BaseDupeFilter:
- @classmethod
- def from_settings(cls, settings: BaseSettings) -> Self:
- ...
-
- def request_seen(self, request: Request) -> bool:
- ...
-
- def open(self) -> Optional[Deferred]:
- ...
-
- def close(self, reason: str) -> Optional[Deferred]:
- ...
-
- def log(self, request: Request, spider: Spider) -> None:
- """Log that a request has been filtered"""
- ...
-
-
-
-class RFPDupeFilter(BaseDupeFilter):
- """Request Fingerprint duplicates filter"""
- def __init__(self, path: Optional[str] = ..., debug: bool = ..., *, fingerprinter: Optional[RequestFingerprinterProtocol] = ...) -> None:
- ...
-
- @classmethod
- def from_settings(cls, settings: BaseSettings, *, fingerprinter: Optional[RequestFingerprinterProtocol] = ...) -> Self:
- ...
-
- @classmethod
- def from_crawler(cls, crawler: Crawler) -> Self:
- ...
-
- def request_seen(self, request: Request) -> bool:
- ...
-
- def request_fingerprint(self, request: Request) -> str:
- ...
-
- def close(self, reason: str) -> None:
- ...
-
- def log(self, request: Request, spider: Spider) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/exceptions.pyi b/typings/scrapy/exceptions.pyi
deleted file mode 100644
index a9f486eb..00000000
--- a/typings/scrapy/exceptions.pyi
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any
-
-"""
-Scrapy core exceptions
-
-These exceptions are documented in docs/topics/exceptions.rst. Please don't add
-new exceptions here without documenting them there.
-"""
-class NotConfigured(Exception):
- """Indicates a missing configuration situation"""
- ...
-
-
-class _InvalidOutput(TypeError):
- """
- Indicates an invalid value has been returned by a middleware's processing method.
- Internal and undocumented, it should not be raised or caught by user code.
- """
- ...
-
-
-class IgnoreRequest(Exception):
- """Indicates a decision was made not to process a request"""
- ...
-
-
-class DontCloseSpider(Exception):
- """Request the spider not to be closed yet"""
- ...
-
-
-class CloseSpider(Exception):
- """Raise this from callbacks to request the spider to be closed"""
- def __init__(self, reason: str = ...) -> None:
- ...
-
-
-
-class StopDownload(Exception):
- """
- Stop the download of the body for a given response.
- The 'fail' boolean parameter indicates whether or not the resulting partial response
- should be handled by the request errback. Note that 'fail' is a keyword-only argument.
- """
- def __init__(self, *, fail: bool = ...) -> None:
- ...
-
-
-
-class DropItem(Exception):
- """Drop item from the item pipeline"""
- ...
-
-
-class NotSupported(Exception):
- """Indicates a feature or method is not supported"""
- ...
-
-
-class UsageError(Exception):
- """To indicate a command-line usage error"""
- def __init__(self, *a: Any, **kw: Any) -> None:
- ...
-
-
-
-class ScrapyDeprecationWarning(Warning):
- """Warning category for deprecated features, since the default
- DeprecationWarning is silenced on Python 2.7+
- """
- ...
-
-
-class ContractFail(AssertionError):
- """Error raised in case of a failing contract"""
- ...
-
-
diff --git a/typings/scrapy/exporters.pyi b/typings/scrapy/exporters.pyi
deleted file mode 100644
index 58131e62..00000000
--- a/typings/scrapy/exporters.pyi
+++ /dev/null
@@ -1,131 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Item Exporters are used to export/serialize items into different formats.
-"""
-__all__ = ["BaseItemExporter", "PprintItemExporter", "PickleItemExporter", "CsvItemExporter", "XmlItemExporter", "JsonLinesItemExporter", "JsonItemExporter", "MarshalItemExporter"]
-class BaseItemExporter:
- def __init__(self, *, dont_fail=..., **kwargs) -> None:
- ...
-
- def export_item(self, item):
- ...
-
- def serialize_field(self, field, name, value):
- ...
-
- def start_exporting(self): # -> None:
- ...
-
- def finish_exporting(self): # -> None:
- ...
-
-
-
-class JsonLinesItemExporter(BaseItemExporter):
- def __init__(self, file, **kwargs) -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
-
-
-class JsonItemExporter(BaseItemExporter):
- def __init__(self, file, **kwargs) -> None:
- ...
-
- def start_exporting(self): # -> None:
- ...
-
- def finish_exporting(self): # -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
-
-
-class XmlItemExporter(BaseItemExporter):
- def __init__(self, file, **kwargs) -> None:
- ...
-
- def start_exporting(self): # -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
- def finish_exporting(self): # -> None:
- ...
-
-
-
-class CsvItemExporter(BaseItemExporter):
- def __init__(self, file, include_headers_line=..., join_multivalued=..., errors=..., **kwargs) -> None:
- ...
-
- def serialize_field(self, field, name, value):
- ...
-
- def export_item(self, item): # -> None:
- ...
-
- def finish_exporting(self): # -> None:
- ...
-
-
-
-class PickleItemExporter(BaseItemExporter):
- def __init__(self, file, protocol=..., **kwargs) -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
-
-
-class MarshalItemExporter(BaseItemExporter):
- """Exports items in a Python-specific binary format (see
- :mod:`marshal`).
-
- :param file: The file-like object to use for exporting the data. Its
- ``write`` method should accept :class:`bytes` (a disk file
- opened in binary mode, a :class:`~io.BytesIO` object, etc)
- """
- def __init__(self, file, **kwargs) -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
-
-
-class PprintItemExporter(BaseItemExporter):
- def __init__(self, file, **kwargs) -> None:
- ...
-
- def export_item(self, item): # -> None:
- ...
-
-
-
-class PythonItemExporter(BaseItemExporter):
- """This is a base class for item exporters that extends
- :class:`BaseItemExporter` with support for nested items.
-
- It serializes items to built-in Python types, so that any serialization
- library (e.g. :mod:`json` or msgpack_) can be used on top of it.
-
- .. _msgpack: https://pypi.org/project/msgpack/
- """
- def serialize_field(self, field, name, value):
- ...
-
- def export_item(self, item): # -> dict[str | Any, Any | None]:
- ...
-
-
-
diff --git a/typings/scrapy/extension.pyi b/typings/scrapy/extension.pyi
deleted file mode 100644
index 2ed1f05b..00000000
--- a/typings/scrapy/extension.pyi
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.middleware import MiddlewareManager
-
-"""
-The Extension Manager
-
-See documentation in docs/topics/extensions.rst
-"""
-class ExtensionManager(MiddlewareManager):
- component_name = ...
-
-
diff --git a/typings/scrapy/extensions/__init__.pyi b/typings/scrapy/extensions/__init__.pyi
deleted file mode 100644
index 006bc274..00000000
--- a/typings/scrapy/extensions/__init__.pyi
+++ /dev/null
@@ -1,4 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
diff --git a/typings/scrapy/http/__init__.pyi b/typings/scrapy/http/__init__.pyi
deleted file mode 100644
index 0209a618..00000000
--- a/typings/scrapy/http/__init__.pyi
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.http.headers import Headers
-from scrapy.http.request import Request
-from scrapy.http.request.form import FormRequest
-from scrapy.http.request.json_request import JsonRequest
-from scrapy.http.request.rpc import XmlRpcRequest
-from scrapy.http.response import Response
-from scrapy.http.response.html import HtmlResponse
-from scrapy.http.response.text import TextResponse
-from scrapy.http.response.xml import XmlResponse
-
-"""
-Module containing all HTTP related classes
-
-Use this module (instead of the more specific ones) when importing Headers,
-Request and Response outside this module.
-"""
diff --git a/typings/scrapy/http/common.pyi b/typings/scrapy/http/common.pyi
deleted file mode 100644
index 91e1e91e..00000000
--- a/typings/scrapy/http/common.pyi
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-def obsolete_setter(setter, attrname): # -> Callable[..., NoReturn]:
- ...
-
diff --git a/typings/scrapy/http/headers.pyi b/typings/scrapy/http/headers.pyi
deleted file mode 100644
index adf19857..00000000
--- a/typings/scrapy/http/headers.pyi
+++ /dev/null
@@ -1,61 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.utils.datatypes import CaselessDict
-
-class Headers(CaselessDict):
- """Case insensitive http headers dictionary"""
- def __init__(self, seq=..., encoding=...) -> None:
- ...
-
- def update(self, seq): # -> None:
- ...
-
- def normkey(self, key): # -> bytes:
- """Normalize key to bytes"""
- ...
-
- def normvalue(self, value): # -> list[bytes]:
- """Normalize values to bytes"""
- ...
-
- def __getitem__(self, key): # -> None:
- ...
-
- def get(self, key, def_val=...): # -> None:
- ...
-
- def getlist(self, key, def_val=...): # -> list[bytes] | list[Any]:
- ...
-
- def setlist(self, key, list_): # -> None:
- ...
-
- def setlistdefault(self, key, default_list=...):
- ...
-
- def appendlist(self, key, value): # -> None:
- ...
-
- def items(self): # -> Generator[tuple[Any, Any | list[bytes] | list[Any]], None, None]:
- ...
-
- def values(self): # -> list[Any | None]:
- ...
-
- def to_string(self): # -> bytes | None:
- ...
-
- def to_unicode_dict(self): # -> CaseInsensitiveDict:
- """Return headers as a CaselessDict with unicode keys
- and unicode values. Multiple values are joined with ','.
- """
- ...
-
- def __copy__(self): # -> Self:
- ...
-
- copy = ...
-
-
diff --git a/typings/scrapy/http/request/__init__.pyi b/typings/scrapy/http/request/__init__.pyi
deleted file mode 100644
index 5843bd31..00000000
--- a/typings/scrapy/http/request/__init__.pyi
+++ /dev/null
@@ -1,116 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import inspect
-import scrapy
-from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
-from w3lib.url import safe_url_string
-from scrapy.http.common import obsolete_setter
-from scrapy.http.headers import Headers
-from scrapy.utils.curl import curl_to_request_kwargs
-from scrapy.utils.python import to_bytes
-from scrapy.utils.trackref import object_ref
-from scrapy.utils.url import escape_ajax
-
-"""
-This module implements the Request class which is used to represent HTTP
-requests in Scrapy.
-
-See documentation in docs/topics/request-response.rst
-"""
-RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
-def NO_CALLBACK(*args, **kwargs):
- """When assigned to the ``callback`` parameter of
- :class:`~scrapy.http.Request`, it indicates that the request is not meant
- to have a spider callback at all.
-
- For example:
-
- .. code-block:: python
-
- Request("https://example.com", callback=NO_CALLBACK)
-
- This value should be used by :ref:`components ` that
- create and handle their own requests, e.g. through
- :meth:`scrapy.core.engine.ExecutionEngine.download`, so that downloader
- middlewares handling such requests can treat them differently from requests
- intended for the :meth:`~scrapy.Spider.parse` callback.
- """
- ...
-
-class Request(object_ref):
- """Represents an HTTP request, which is usually generated in a Spider and
- executed by the Downloader, thus generating a :class:`Response`.
- """
- attributes: Tuple[str, ...] = ...
- def __init__(self, url: str, callback: Optional[Callable] = ..., method: str = ..., headers: Optional[dict] = ..., body: Optional[Union[bytes, str]] = ..., cookies: Optional[Union[dict, List[dict]]] = ..., meta: Optional[dict] = ..., encoding: str = ..., priority: int = ..., dont_filter: bool = ..., errback: Optional[Callable] = ..., flags: Optional[List[str]] = ..., cb_kwargs: Optional[dict] = ...) -> None:
- ...
-
- @property
- def cb_kwargs(self) -> dict:
- ...
-
- @property
- def meta(self) -> dict:
- ...
-
- url = ...
- body = ...
- @property
- def encoding(self) -> str:
- ...
-
- def __repr__(self) -> str:
- ...
-
- def copy(self) -> Request:
- ...
-
- def replace(self, *args, **kwargs) -> Request:
- """Create a new Request with the same attributes except for those given new values"""
- ...
-
- @classmethod
- def from_curl(cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = ..., **kwargs) -> RequestTypeVar:
- """Create a Request object from a string containing a `cURL
- `_ command. It populates the HTTP method, the
- URL, the headers, the cookies and the body. It accepts the same
- arguments as the :class:`Request` class, taking preference and
- overriding the values of the same arguments contained in the cURL
- command.
-
- Unrecognized options are ignored by default. To raise an error when
- finding unknown options call this method by passing
- ``ignore_unknown_options=False``.
-
- .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
- subclasses, such as :class:`~scrapy.http.JSONRequest`, or
- :class:`~scrapy.http.XmlRpcRequest`, as well as having
- :ref:`downloader middlewares `
- and
- :ref:`spider middlewares `
- enabled, such as
- :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`,
- :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`,
- or
- :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
- may modify the :class:`~scrapy.http.Request` object.
-
- To translate a cURL command into a Scrapy request,
- you may use `curl2scrapy `_.
- """
- ...
-
- def to_dict(self, *, spider: Optional[scrapy.Spider] = ...) -> dict:
- """Return a dictionary containing the Request's data.
-
- Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
-
- If a spider is given, this method will try to find out the name of the spider methods used as callback
- and errback and include them in the output dict, raising an exception if they cannot be found.
- """
- ...
-
-
-
diff --git a/typings/scrapy/http/request/form.pyi b/typings/scrapy/http/request/form.pyi
deleted file mode 100644
index b4b45923..00000000
--- a/typings/scrapy/http/request/form.pyi
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
-from scrapy.http.request import Request
-from scrapy.http.response.text import TextResponse
-
-"""
-This module implements the FormRequest class which is a more convenient class
-(than Request) to generate Requests based on form data.
-
-See documentation in docs/topics/request-response.rst
-"""
-FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
-FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
-FormdataType = Optional[Union[dict, List[FormdataKVType]]]
-class FormRequest(Request):
- valid_form_methods = ...
- def __init__(self, *args, formdata: FormdataType = ..., **kwargs) -> None:
- ...
-
- @classmethod
- def from_response(cls: Type[FormRequestTypeVar], response: TextResponse, formname: Optional[str] = ..., formid: Optional[str] = ..., formnumber: int = ..., formdata: FormdataType = ..., clickdata: Optional[dict] = ..., dont_click: bool = ..., formxpath: Optional[str] = ..., formcss: Optional[str] = ..., **kwargs) -> FormRequestTypeVar:
- ...
-
-
-
diff --git a/typings/scrapy/http/request/json_request.pyi b/typings/scrapy/http/request/json_request.pyi
deleted file mode 100644
index 7f912bac..00000000
--- a/typings/scrapy/http/request/json_request.pyi
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Optional, Tuple
-from scrapy.http.request import Request
-
-"""
-This module implements the JsonRequest class which is a more convenient class
-(than Request) to generate JSON Requests.
-
-See documentation in docs/topics/request-response.rst
-"""
-class JsonRequest(Request):
- attributes: Tuple[str, ...] = ...
- def __init__(self, *args, dumps_kwargs: Optional[dict] = ..., **kwargs) -> None:
- ...
-
- @property
- def dumps_kwargs(self) -> dict:
- ...
-
- def replace(self, *args, **kwargs) -> Request:
- ...
-
-
-
-JSONRequest = ...
diff --git a/typings/scrapy/http/request/rpc.pyi b/typings/scrapy/http/request/rpc.pyi
deleted file mode 100644
index 9cf84260..00000000
--- a/typings/scrapy/http/request/rpc.pyi
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Optional
-from scrapy.http.request import Request
-
-"""
-This module implements the XmlRpcRequest class which is a more convenient class
-(that Request) to generate xml-rpc requests.
-
-See documentation in docs/topics/request-response.rst
-"""
-DUMPS_ARGS = ...
-class XmlRpcRequest(Request):
- def __init__(self, *args, encoding: Optional[str] = ..., **kwargs) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/http/response/__init__.pyi b/typings/scrapy/http/response/__init__.pyi
deleted file mode 100644
index d6106d35..00000000
--- a/typings/scrapy/http/response/__init__.pyi
+++ /dev/null
@@ -1,111 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Generator, Tuple
-from urllib.parse import urljoin
-from scrapy.exceptions import NotSupported
-from scrapy.http.common import obsolete_setter
-from scrapy.http.headers import Headers
-from scrapy.http.request import Request
-from scrapy.link import Link
-from scrapy.utils.trackref import object_ref
-
-"""
-This module implements the Response class which is used to represent HTTP
-responses in Scrapy.
-
-See documentation in docs/topics/request-response.rst
-"""
-class Response(object_ref):
- """An object that represents an HTTP response, which is usually
- downloaded (by the Downloader) and fed to the Spiders for processing.
- """
- attributes: Tuple[str, ...] = ...
- def __init__(self, url: str, status=..., headers=..., body=..., flags=..., request=..., certificate=..., ip_address=..., protocol=...) -> None:
- ...
-
- @property
- def cb_kwargs(self):
- ...
-
- @property
- def meta(self):
- ...
-
- url = ...
- body = ...
- def __repr__(self): # -> str:
- ...
-
- def copy(self):
- """Return a copy of this Response"""
- ...
-
- def replace(self, *args, **kwargs):
- """Create a new Response with the same attributes except for those given new values"""
- ...
-
- def urljoin(self, url): # -> Any:
- """Join this Response's url with a possible relative url to form an
- absolute interpretation of the latter."""
- ...
-
- @property
- def text(self):
- """For subclasses of TextResponse, this will return the body
- as str
- """
- ...
-
- def css(self, *a, **kw):
- """Shortcut method implemented only by responses whose content
- is text (subclasses of TextResponse).
- """
- ...
-
- def jmespath(self, *a, **kw):
- """Shortcut method implemented only by responses whose content
- is text (subclasses of TextResponse).
- """
- ...
-
- def xpath(self, *a, **kw):
- """Shortcut method implemented only by responses whose content
- is text (subclasses of TextResponse).
- """
- ...
-
- def follow(self, url, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Request:
- """
- Return a :class:`~.Request` instance to follow a link ``url``.
- It accepts the same arguments as ``Request.__init__`` method,
- but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
- not only an absolute URL.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
- method which supports selectors in addition to absolute/relative URLs
- and Link objects.
-
- .. versionadded:: 2.0
- The *flags* parameter.
- """
- ...
-
- def follow_all(self, urls, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Generator[Request, None, None]:
- """
- .. versionadded:: 2.0
-
- Return an iterable of :class:`~.Request` instances to follow all links
- in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
- but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
- not only absolute URLs.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
- method which supports selectors in addition to absolute/relative URLs
- and Link objects.
- """
- ...
-
-
-
diff --git a/typings/scrapy/http/response/html.pyi b/typings/scrapy/http/response/html.pyi
deleted file mode 100644
index dc9e6637..00000000
--- a/typings/scrapy/http/response/html.pyi
+++ /dev/null
@@ -1,16 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.http.response.text import TextResponse
-
-"""
-This module implements the HtmlResponse class which adds encoding
-discovering through HTML encoding declarations to the TextResponse class.
-
-See documentation in docs/topics/request-response.rst
-"""
-class HtmlResponse(TextResponse):
- ...
-
-
diff --git a/typings/scrapy/http/response/text.pyi b/typings/scrapy/http/response/text.pyi
deleted file mode 100644
index 3f23e40c..00000000
--- a/typings/scrapy/http/response/text.pyi
+++ /dev/null
@@ -1,113 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Generator, TYPE_CHECKING, Tuple
-from scrapy.http import Request
-from scrapy.http.response import Response
-
-"""
-This module implements the TextResponse class which adds encoding handling and
-discovering (through HTTP headers) to base Response class.
-
-See documentation in docs/topics/request-response.rst
-"""
-if TYPE_CHECKING:
- ...
-_NONE = ...
-class TextResponse(Response):
- _DEFAULT_ENCODING = ...
- _cached_decoded_json = ...
- attributes: Tuple[str, ...] = ...
- def __init__(self, *args: Any, **kwargs: Any) -> None:
- ...
-
- @property
- def encoding(self): # -> Any | str:
- ...
-
- def json(self): # -> Any | object:
- """
- .. versionadded:: 2.2
-
- Deserialize a JSON document to a Python object.
- """
- ...
-
- @property
- def text(self) -> str:
- """Body as unicode"""
- ...
-
- def urljoin(self, url): # -> str:
- """Join this Response's url with a possible relative url to form an
- absolute interpretation of the latter."""
- ...
-
- @property
- def selector(self): # -> Selector:
- ...
-
- def jmespath(self, query, **kwargs): # -> SelectorList[Selector]:
- ...
-
- def xpath(self, query, **kwargs): # -> SelectorList[Selector]:
- ...
-
- def css(self, query): # -> SelectorList[Selector]:
- ...
-
- def follow(self, url, callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=...) -> Request:
- """
- Return a :class:`~.Request` instance to follow a link ``url``.
- It accepts the same arguments as ``Request.__init__`` method,
- but ``url`` can be not only an absolute URL, but also
-
- * a relative URL
- * a :class:`~scrapy.link.Link` object, e.g. the result of
- :ref:`topics-link-extractors`
- * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g.
- ``response.css('a.my_link')[0]``
- * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
- ``response.css('a::attr(href)')[0]`` or
- ``response.xpath('//img/@src')[0]``
-
- See :ref:`response-follow-example` for usage examples.
- """
- ...
-
- def follow_all(self, urls=..., callback=..., method=..., headers=..., body=..., cookies=..., meta=..., encoding=..., priority=..., dont_filter=..., errback=..., cb_kwargs=..., flags=..., css=..., xpath=...) -> Generator[Request, None, None]:
- """
- A generator that produces :class:`~.Request` instances to follow all
- links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
- ``__init__`` method, except that each ``urls`` element does not need to be
- an absolute URL, it can be any of the following:
-
- * a relative URL
- * a :class:`~scrapy.link.Link` object, e.g. the result of
- :ref:`topics-link-extractors`
- * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g.
- ``response.css('a.my_link')[0]``
- * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
- ``response.css('a::attr(href)')[0]`` or
- ``response.xpath('//img/@src')[0]``
-
- In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
- within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
-
- Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
- using the ``css`` or ``xpath`` parameters, this method will not produce requests for
- selectors from which links cannot be obtained (for instance, anchor tags without an
- ``href`` attribute)
- """
- ...
-
-
-
-class _InvalidSelector(ValueError):
- """
- Raised when a URL cannot be obtained from a Selector
- """
- ...
-
-
diff --git a/typings/scrapy/http/response/xml.pyi b/typings/scrapy/http/response/xml.pyi
deleted file mode 100644
index 27a3928e..00000000
--- a/typings/scrapy/http/response/xml.pyi
+++ /dev/null
@@ -1,16 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.http.response.text import TextResponse
-
-"""
-This module implements the XmlResponse class which adds encoding
-discovering through XML encoding declarations to the TextResponse class.
-
-See documentation in docs/topics/request-response.rst
-"""
-class XmlResponse(TextResponse):
- ...
-
-
diff --git a/typings/scrapy/interfaces.pyi b/typings/scrapy/interfaces.pyi
deleted file mode 100644
index e42719de..00000000
--- a/typings/scrapy/interfaces.pyi
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from zope.interface import Interface
-
-class ISpiderLoader(Interface):
- def from_settings(settings): # -> None:
- """Return an instance of the class for the given settings"""
- ...
-
- def load(spider_name): # -> None:
- """Return the Spider class for the given spider name. If the spider
- name is not found, it must raise a KeyError."""
- ...
-
- def list(): # -> None:
- """Return a list with the names of all spiders available in the
- project"""
- ...
-
- def find_by_request(request): # -> None:
- """Return the list of spiders names that can handle the given request"""
- ...
-
-
-
diff --git a/typings/scrapy/item.pyi b/typings/scrapy/item.pyi
deleted file mode 100644
index adeacc8e..00000000
--- a/typings/scrapy/item.pyi
+++ /dev/null
@@ -1,92 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from abc import ABCMeta
-from collections.abc import MutableMapping
-from typing import Dict
-from scrapy.utils.trackref import object_ref
-
-"""
-Scrapy Item
-
-See documentation in docs/topics/item.rst
-"""
-class Field(dict):
- """Container of field metadata"""
- ...
-
-
-class ItemMeta(ABCMeta):
- """Metaclass_ of :class:`Item` that handles field definitions.
-
- .. _metaclass: https://realpython.com/python-metaclasses
- """
- def __new__(mcs, class_name, bases, attrs): # -> Self:
- ...
-
-
-
-class Item(MutableMapping, object_ref, metaclass=ItemMeta):
- """
- Base class for scraped items.
-
- In Scrapy, an object is considered an ``item`` if it is an instance of either
- :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
- spider callback is evaluated, only instances of :class:`Item` or
- :class:`dict` are passed to :ref:`item pipelines `.
-
- If you need instances of a custom class to be considered items by Scrapy,
- you must inherit from either :class:`Item` or :class:`dict`.
-
- Items must declare :class:`Field` attributes, which are processed and stored
- in the ``fields`` attribute. This restricts the set of allowed field names
- and prevents typos, raising ``KeyError`` when referring to undefined fields.
- Additionally, fields can be used to define metadata and control the way
- data is processed internally. Please refer to the :ref:`documentation
- about fields ` for additional information.
-
- Unlike instances of :class:`dict`, instances of :class:`Item` may be
- :ref:`tracked ` to debug memory leaks.
- """
- fields: Dict[str, Field]
- def __init__(self, *args, **kwargs) -> None:
- ...
-
- def __getitem__(self, key):
- ...
-
- def __setitem__(self, key, value): # -> None:
- ...
-
- def __delitem__(self, key): # -> None:
- ...
-
- def __getattr__(self, name):
- ...
-
- def __setattr__(self, name, value): # -> None:
- ...
-
- def __len__(self): # -> int:
- ...
-
- def __iter__(self): # -> Iterator[Any]:
- ...
-
- __hash__ = ...
- def keys(self): # -> dict_keys[Any, Any]:
- ...
-
- def __repr__(self): # -> str:
- ...
-
- def copy(self): # -> Self:
- ...
-
- def deepcopy(self): # -> Self:
- """Return a :func:`~copy.deepcopy` of this item."""
- ...
-
-
-
diff --git a/typings/scrapy/link.pyi b/typings/scrapy/link.pyi
deleted file mode 100644
index 8b69c2e1..00000000
--- a/typings/scrapy/link.pyi
+++ /dev/null
@@ -1,44 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any
-
-"""
-This module defines the Link object used in Link extractors.
-
-For actual link extractors implementation see scrapy.linkextractors, or
-its documentation in: docs/topics/link-extractors.rst
-"""
-class Link:
- """Link objects represent an extracted link by the LinkExtractor.
-
- Using the anchor tag sample below to illustrate the parameters::
-
- Dont follow this one
-
- :param url: the absolute url being linked to in the anchor tag.
- From the sample, this is ``https://example.com/nofollow.html``.
-
- :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
-
- :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
-
- :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
- of the anchor tag.
- """
- __slots__ = ...
- def __init__(self, url: str, text: str = ..., fragment: str = ..., nofollow: bool = ...) -> None:
- ...
-
- def __eq__(self, other: Any) -> bool:
- ...
-
- def __hash__(self) -> int:
- ...
-
- def __repr__(self) -> str:
- ...
-
-
-
diff --git a/typings/scrapy/linkextractors/__init__.pyi b/typings/scrapy/linkextractors/__init__.pyi
deleted file mode 100644
index 79b49626..00000000
--- a/typings/scrapy/linkextractors/__init__.pyi
+++ /dev/null
@@ -1,16 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import re
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
-
-"""
-scrapy.linkextractors
-
-This package contains a collection of Link Extractors.
-
-For more info see docs/topics/link-extractors.rst
-"""
-IGNORED_EXTENSIONS = ...
-_re_type = ...
diff --git a/typings/scrapy/linkextractors/lxmlhtml.pyi b/typings/scrapy/linkextractors/lxmlhtml.pyi
deleted file mode 100644
index d8979d82..00000000
--- a/typings/scrapy/linkextractors/lxmlhtml.pyi
+++ /dev/null
@@ -1,41 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Link extractor based on lxml.html
-"""
-logger = ...
-XHTML_NAMESPACE = ...
-_collect_string_content = ...
-class LxmlParserLinkExtractor:
- def __init__(self, tag=..., attr=..., process=..., unique=..., strip=..., canonicalized=...) -> None:
- ...
-
- def extract_links(self, response): # -> list[Any]:
- ...
-
-
-
-class LxmlLinkExtractor:
- _csstranslator = ...
- def __init__(self, allow=..., deny=..., allow_domains=..., deny_domains=..., restrict_xpaths=..., tags=..., attrs=..., canonicalize=..., unique=..., process_value=..., deny_extensions=..., restrict_css=..., strip=..., restrict_text=...) -> None:
- ...
-
- def matches(self, url): # -> bool:
- ...
-
- def extract_links(self, response): # -> list[Any]:
- """Returns a list of :class:`~scrapy.link.Link` objects from the
- specified :class:`response `.
-
- Only links that match the settings passed to the ``__init__`` method of
- the link extractor are returned.
-
- Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
- otherwise they are returned.
- """
- ...
-
-
-
diff --git a/typings/scrapy/loader/__init__.pyi b/typings/scrapy/loader/__init__.pyi
deleted file mode 100644
index be8853cd..00000000
--- a/typings/scrapy/loader/__init__.pyi
+++ /dev/null
@@ -1,88 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import itemloaders
-from scrapy.item import Item
-from scrapy.selector import Selector
-
-"""
-Item Loader
-
-See documentation in docs/topics/loaders.rst
-"""
-class ItemLoader(itemloaders.ItemLoader):
- """
- A user-friendly abstraction to populate an :ref:`item ` with data
- by applying :ref:`field processors ` to scraped data.
- When instantiated with a ``selector`` or a ``response`` it supports
- data extraction from web pages using :ref:`selectors `.
-
- :param item: The item instance to populate using subsequent calls to
- :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
- or :meth:`~ItemLoader.add_value`.
- :type item: scrapy.item.Item
-
- :param selector: The selector to extract data from, when using the
- :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
- :meth:`replace_css` method.
- :type selector: :class:`~scrapy.selector.Selector` object
-
- :param response: The response used to construct the selector using the
- :attr:`default_selector_class`, unless the selector argument is given,
- in which case this argument is ignored.
- :type response: :class:`~scrapy.http.Response` object
-
- If no item is given, one is instantiated automatically using the class in
- :attr:`default_item_class`.
-
- The item, selector, response and remaining keyword arguments are
- assigned to the Loader context (accessible through the :attr:`context` attribute).
-
- .. attribute:: item
-
- The item object being parsed by this Item Loader.
- This is mostly used as a property so, when attempting to override this
- value, you may want to check out :attr:`default_item_class` first.
-
- .. attribute:: context
-
- The currently active :ref:`Context ` of this Item Loader.
-
- .. attribute:: default_item_class
-
- An :ref:`item ` class (or factory), used to instantiate
- items when not given in the ``__init__`` method.
-
- .. attribute:: default_input_processor
-
- The default input processor to use for those fields which don't specify
- one.
-
- .. attribute:: default_output_processor
-
- The default output processor to use for those fields which don't specify
- one.
-
- .. attribute:: default_selector_class
-
- The class used to construct the :attr:`selector` of this
- :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
- If a selector is given in the ``__init__`` method this attribute is ignored.
- This attribute is sometimes overridden in subclasses.
-
- .. attribute:: selector
-
- The :class:`~scrapy.selector.Selector` object to extract data from.
- It's either the selector given in the ``__init__`` method or one created from
- the response given in the ``__init__`` method using the
- :attr:`default_selector_class`. This attribute is meant to be
- read-only.
- """
- default_item_class = Item
- default_selector_class = Selector
- def __init__(self, item=..., selector=..., response=..., parent=..., **context) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/logformatter.pyi b/typings/scrapy/logformatter.pyi
deleted file mode 100644
index 94f15400..00000000
--- a/typings/scrapy/logformatter.pyi
+++ /dev/null
@@ -1,98 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Optional, TYPE_CHECKING, Union
-from twisted.python.failure import Failure
-from scrapy import Request, Spider
-from scrapy.http import Response
-from typing_extensions import Self
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-SCRAPEDMSG = ...
-DROPPEDMSG = ...
-CRAWLEDMSG = ...
-ITEMERRORMSG = ...
-SPIDERERRORMSG = ...
-DOWNLOADERRORMSG_SHORT = ...
-DOWNLOADERRORMSG_LONG = ...
-class LogFormatter:
- """Class for generating log messages for different actions.
-
- All methods must return a dictionary listing the parameters ``level``, ``msg``
- and ``args`` which are going to be used for constructing the log message when
- calling ``logging.log``.
-
- Dictionary keys for the method outputs:
-
- * ``level`` is the log level for that action, you can use those from the
- `python logging library `_ :
- ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
- and ``logging.CRITICAL``.
- * ``msg`` should be a string that can contain different formatting placeholders.
- This string, formatted with the provided ``args``, is going to be the long message
- for that action.
- * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
- The final log message is computed as ``msg % args``.
-
- Users can define their own ``LogFormatter`` class if they want to customize how
- each action is logged or if they want to omit it entirely. In order to omit
- logging an action the method must return ``None``.
-
- Here is an example on how to create a custom log formatter to lower the severity level of
- the log message when an item is dropped from the pipeline::
-
- class PoliteLogFormatter(logformatter.LogFormatter):
- def dropped(self, item, exception, response, spider):
- return {
- 'level': logging.INFO, # lowering the level from logging.WARNING
- 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
- 'args': {
- 'exception': exception,
- 'item': item,
- }
- }
- """
- def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
- """Logs a message when the crawler finds a webpage."""
- ...
-
- def scraped(self, item: Any, response: Union[Response, Failure], spider: Spider) -> dict:
- """Logs a message when an item is scraped by a spider."""
- ...
-
- def dropped(self, item: Any, exception: BaseException, response: Response, spider: Spider) -> dict:
- """Logs a message when an item is dropped while it is passing through the item pipeline."""
- ...
-
- def item_error(self, item: Any, exception: BaseException, response: Response, spider: Spider) -> dict:
- """Logs a message when an item causes an error while it is passing
- through the item pipeline.
-
- .. versionadded:: 2.0
- """
- ...
-
- def spider_error(self, failure: Failure, request: Request, response: Union[Response, Failure], spider: Spider) -> dict:
- """Logs an error message from a spider.
-
- .. versionadded:: 2.0
- """
- ...
-
- def download_error(self, failure: Failure, request: Request, spider: Spider, errmsg: Optional[str] = ...) -> dict:
- """Logs a download error message from a spider (typically coming from
- the engine).
-
- .. versionadded:: 2.0
- """
- ...
-
- @classmethod
- def from_crawler(cls, crawler: Crawler) -> Self:
- ...
-
-
-
diff --git a/typings/scrapy/mail.pyi b/typings/scrapy/mail.pyi
deleted file mode 100644
index dcfd80f2..00000000
--- a/typings/scrapy/mail.pyi
+++ /dev/null
@@ -1,24 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Mail sending helpers
-
-See documentation in docs/topics/email.rst
-"""
-logger = ...
-COMMASPACE = ...
-class MailSender:
- def __init__(self, smtphost=..., mailfrom=..., smtpuser=..., smtppass=..., smtpport=..., smtptls=..., smtpssl=..., debug=...) -> None:
- ...
-
- @classmethod
- def from_settings(cls, settings): # -> Self:
- ...
-
- def send(self, to, subject, body, cc=..., attachs=..., mimetype=..., charset=..., _callback=...): # -> Deferred[Any] | None:
- ...
-
-
-
diff --git a/typings/scrapy/middleware.pyi b/typings/scrapy/middleware.pyi
deleted file mode 100644
index f7c23a20..00000000
--- a/typings/scrapy/middleware.pyi
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Optional, TYPE_CHECKING
-from twisted.internet.defer import Deferred
-from scrapy import Spider
-from scrapy.settings import Settings
-from typing_extensions import Self
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-logger = ...
-class MiddlewareManager:
- """Base class for implementing middleware managers"""
- component_name = ...
- def __init__(self, *middlewares: Any) -> None:
- ...
-
- @classmethod
- def from_settings(cls, settings: Settings, crawler: Optional[Crawler] = ...) -> Self:
- ...
-
- @classmethod
- def from_crawler(cls, crawler: Crawler) -> Self:
- ...
-
- def open_spider(self, spider: Spider) -> Deferred:
- ...
-
- def close_spider(self, spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/pipelines/__init__.pyi b/typings/scrapy/pipelines/__init__.pyi
deleted file mode 100644
index 78efa43a..00000000
--- a/typings/scrapy/pipelines/__init__.pyi
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, List
-from twisted.internet.defer import Deferred
-from scrapy import Spider
-from scrapy.middleware import MiddlewareManager
-from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import deferred_f_from_coro_f
-
-"""
-Item pipeline
-
-See documentation in docs/item-pipeline.rst
-"""
-class ItemPipelineManager(MiddlewareManager):
- component_name = ...
- def process_item(self, item: Any, spider: Spider) -> Deferred:
- ...
-
-
-
diff --git a/typings/scrapy/pqueues.pyi b/typings/scrapy/pqueues.pyi
deleted file mode 100644
index b0918e7a..00000000
--- a/typings/scrapy/pqueues.pyi
+++ /dev/null
@@ -1,122 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-logger = ...
-class ScrapyPriorityQueue:
- """A priority queue implemented using multiple internal queues (typically,
- FIFO queues). It uses one internal queue for each priority value. The internal
- queue must implement the following methods:
-
- * push(obj)
- * pop()
- * close()
- * __len__()
-
- Optionally, the queue could provide a ``peek`` method, that should return the
- next object to be returned by ``pop``, but without removing it from the queue.
-
- ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
- argument, which is a class used to instantiate a new (internal) queue when
- a new priority is allocated.
-
- Only integer priorities should be used. Lower numbers are higher
- priorities.
-
- startprios is a sequence of priorities to start with. If the queue was
- previously closed leaving some priority buckets non-empty, those priorities
- should be passed in startprios.
-
- """
- @classmethod
- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=...): # -> Self:
- ...
-
- def __init__(self, crawler, downstream_queue_cls, key, startprios=...) -> None:
- ...
-
- def init_prios(self, startprios): # -> None:
- ...
-
- def qfactory(self, key):
- ...
-
- def priority(self, request):
- ...
-
- def push(self, request): # -> None:
- ...
-
- def pop(self): # -> None:
- ...
-
- def peek(self): # -> None:
- """Returns the next object to be returned by :meth:`pop`,
- but without removing it from the queue.
-
- Raises :exc:`NotImplementedError` if the underlying queue class does
- not implement a ``peek`` method, which is optional for queues.
- """
- ...
-
- def close(self): # -> list[Any]:
- ...
-
- def __len__(self): # -> int:
- ...
-
-
-
-class DownloaderInterface:
- def __init__(self, crawler) -> None:
- ...
-
- def stats(self, possible_slots): # -> list[tuple[int, Any]]:
- ...
-
- def get_slot_key(self, request):
- ...
-
-
-
-class DownloaderAwarePriorityQueue:
- """PriorityQueue which takes Downloader activity into account:
- domains (slots) with the least amount of active downloads are dequeued
- first.
- """
- @classmethod
- def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=...): # -> Self:
- ...
-
- def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=...) -> None:
- ...
-
- def pqfactory(self, slot, startprios=...): # -> ScrapyPriorityQueue:
- ...
-
- def pop(self): # -> None:
- ...
-
- def push(self, request): # -> None:
- ...
-
- def peek(self): # -> None:
- """Returns the next object to be returned by :meth:`pop`,
- but without removing it from the queue.
-
- Raises :exc:`NotImplementedError` if the underlying queue class does
- not implement a ``peek`` method, which is optional for queues.
- """
- ...
-
- def close(self): # -> dict[Any, Any]:
- ...
-
- def __len__(self): # -> int:
- ...
-
- def __contains__(self, slot): # -> bool:
- ...
-
-
-
diff --git a/typings/scrapy/resolver.pyi b/typings/scrapy/resolver.pyi
deleted file mode 100644
index ba235dad..00000000
--- a/typings/scrapy/resolver.pyi
+++ /dev/null
@@ -1,78 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any
-from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
-from zope.interface.declarations import implementer, provider
-from scrapy.utils.datatypes import LocalCache
-
-dnscache: LocalCache[str, Any] = ...
-@implementer(IResolverSimple)
-class CachingThreadedResolver(ThreadedResolver):
- """
- Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
- """
- def __init__(self, reactor, cache_size, timeout) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, reactor): # -> Self:
- ...
-
- def install_on_reactor(self): # -> None:
- ...
-
- def getHostByName(self, name: str, timeout=...): # -> Deferred[Any] | Deferred[str]:
- ...
-
-
-
-@implementer(IHostResolution)
-class HostResolution:
- def __init__(self, name) -> None:
- ...
-
- def cancel(self):
- ...
-
-
-
-@provider(IResolutionReceiver)
-class _CachingResolutionReceiver:
- def __init__(self, resolutionReceiver, hostName) -> None:
- ...
-
- def resolutionBegan(self, resolution): # -> None:
- ...
-
- def addressResolved(self, address): # -> None:
- ...
-
- def resolutionComplete(self): # -> None:
- ...
-
-
-
-@implementer(IHostnameResolver)
-class CachingHostnameResolver:
- """
- Experimental caching resolver. Resolves IPv4 and IPv6 addresses,
- does not support setting a timeout value for DNS requests.
- """
- def __init__(self, reactor, cache_size) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, reactor): # -> Self:
- ...
-
- def install_on_reactor(self): # -> None:
- ...
-
- def resolveHostName(self, resolutionReceiver, hostName: str, portNumber=..., addressTypes=..., transportSemantics=...):
- ...
-
-
-
diff --git a/typings/scrapy/responsetypes.pyi b/typings/scrapy/responsetypes.pyi
deleted file mode 100644
index 3e13c24f..00000000
--- a/typings/scrapy/responsetypes.pyi
+++ /dev/null
@@ -1,52 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Mapping, Optional, Type, Union
-from scrapy.http import Response
-
-"""
-This module implements a class which returns the appropriate Response class
-based on different criteria.
-"""
-class ResponseTypes:
- CLASSES = ...
- def __init__(self) -> None:
- ...
-
- def from_mimetype(self, mimetype: str) -> Type[Response]:
- """Return the most appropriate Response class for the given mimetype"""
- ...
-
- def from_content_type(self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = ...) -> Type[Response]:
- """Return the most appropriate Response class from an HTTP Content-Type
- header"""
- ...
-
- def from_content_disposition(self, content_disposition: Union[str, bytes]) -> Type[Response]:
- ...
-
- def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]:
- """Return the most appropriate Response class by looking at the HTTP
- headers"""
- ...
-
- def from_filename(self, filename: str) -> Type[Response]:
- """Return the most appropriate Response class from a file name"""
- ...
-
- def from_body(self, body: bytes) -> Type[Response]:
- """Try to guess the appropriate response based on the body content.
- This method is a bit magic and could be improved in the future, but
- it's not meant to be used except for special cases where response types
- cannot be guess using more straightforward methods."""
- ...
-
- def from_args(self, headers: Optional[Mapping[bytes, bytes]] = ..., url: Optional[str] = ..., filename: Optional[str] = ..., body: Optional[bytes] = ...) -> Type[Response]:
- """Guess the most appropriate Response class based on
- the given arguments."""
- ...
-
-
-
-responsetypes = ...
diff --git a/typings/scrapy/robotstxt.pyi b/typings/scrapy/robotstxt.pyi
deleted file mode 100644
index 28f28d02..00000000
--- a/typings/scrapy/robotstxt.pyi
+++ /dev/null
@@ -1,91 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from abc import ABCMeta, abstractmethod
-
-logger = ...
-def decode_robotstxt(robotstxt_body, spider, to_native_str_type=...): # -> str:
- ...
-
-class RobotParser(metaclass=ABCMeta):
- @classmethod
- @abstractmethod
- def from_crawler(cls, crawler, robotstxt_body): # -> None:
- """Parse the content of a robots.txt_ file as bytes. This must be a class method.
- It must return a new instance of the parser backend.
-
- :param crawler: crawler which made the request
- :type crawler: :class:`~scrapy.crawler.Crawler` instance
-
- :param robotstxt_body: content of a robots.txt_ file.
- :type robotstxt_body: bytes
- """
- ...
-
- @abstractmethod
- def allowed(self, url, user_agent): # -> None:
- """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
-
- :param url: Absolute URL
- :type url: str
-
- :param user_agent: User agent
- :type user_agent: str
- """
- ...
-
-
-
-class PythonRobotParser(RobotParser):
- def __init__(self, robotstxt_body, spider) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, robotstxt_body): # -> Self:
- ...
-
- def allowed(self, url, user_agent): # -> bool:
- ...
-
-
-
-class ReppyRobotParser(RobotParser):
- def __init__(self, robotstxt_body, spider) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, robotstxt_body): # -> Self:
- ...
-
- def allowed(self, url, user_agent):
- ...
-
-
-
-class RerpRobotParser(RobotParser):
- def __init__(self, robotstxt_body, spider) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, robotstxt_body): # -> Self:
- ...
-
- def allowed(self, url, user_agent):
- ...
-
-
-
-class ProtegoRobotParser(RobotParser):
- def __init__(self, robotstxt_body, spider) -> None:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, robotstxt_body): # -> Self:
- ...
-
- def allowed(self, url, user_agent): # -> Literal[True]:
- ...
-
-
-
diff --git a/typings/scrapy/selector/__init__.pyi b/typings/scrapy/selector/__init__.pyi
deleted file mode 100644
index ae51a54a..00000000
--- a/typings/scrapy/selector/__init__.pyi
+++ /dev/null
@@ -1,9 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.selector.unified import Selector, SelectorList
-
-"""
-Selectors
-"""
diff --git a/typings/scrapy/selector/unified.pyi b/typings/scrapy/selector/unified.pyi
deleted file mode 100644
index e329f020..00000000
--- a/typings/scrapy/selector/unified.pyi
+++ /dev/null
@@ -1,59 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Optional
-from parsel import Selector as _ParselSelector
-from scrapy.http import TextResponse
-from scrapy.utils.trackref import object_ref
-
-"""
-XPath selectors based on lxml
-"""
-__all__ = ["Selector", "SelectorList"]
-_NOT_SET = ...
-class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
- """
- The :class:`SelectorList` class is a subclass of the builtin ``list``
- class, which provides a few additional methods.
- """
- ...
-
-
-class Selector(_ParselSelector, object_ref):
- """
- An instance of :class:`Selector` is a wrapper over response to select
- certain parts of its content.
-
- ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
- :class:`~scrapy.http.XmlResponse` object that will be used for selecting
- and extracting data.
-
- ``text`` is a unicode string or utf-8 encoded text for cases when a
- ``response`` isn't available. Using ``text`` and ``response`` together is
- undefined behavior.
-
- ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
- or ``None`` (default).
-
- If ``type`` is ``None``, the selector automatically chooses the best type
- based on ``response`` type (see below), or defaults to ``"html"`` in case it
- is used together with ``text``.
-
- If ``type`` is ``None`` and a ``response`` is passed, the selector type is
- inferred from the response type as follows:
-
- * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
- * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
- * ``"html"`` for anything else
-
- Otherwise, if ``type`` is set, the selector type will be forced and no
- detection will occur.
- """
- __slots__ = ...
- selectorlist_cls = SelectorList
- def __init__(self, response: Optional[TextResponse] = ..., text: Optional[str] = ..., type: Optional[str] = ..., root: Optional[Any] = ..., **kwargs: Any) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/settings/__init__.pyi b/typings/scrapy/settings/__init__.pyi
deleted file mode 100644
index 84fc9f45..00000000
--- a/typings/scrapy/settings/__init__.pyi
+++ /dev/null
@@ -1,371 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import copy
-import json
-from __future__ import annotations
-from importlib import import_module
-from pprint import pformat
-from types import ModuleType
-from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, TYPE_CHECKING, Tuple, Union, cast
-from scrapy.settings import default_settings
-from _typeshed import SupportsItems
-from typing_extensions import Self
-
-_SettingsKeyT = Union[bool, float, int, str, None]
-if TYPE_CHECKING:
- _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None]
-SETTINGS_PRIORITIES: Dict[str, int] = ...
-def get_settings_priority(priority: Union[int, str]) -> int:
- """
- Small helper function that looks up a given string priority in the
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
- numerical value, or directly returns a given numerical priority.
- """
- ...
-
-class SettingsAttribute:
- """Class for storing data related to settings attributes.
-
- This class is intended for internal usage, you should try Settings class
- for settings configuration, not this one.
- """
- def __init__(self, value: Any, priority: int) -> None:
- ...
-
- def set(self, value: Any, priority: int) -> None:
- """Sets value if priority is higher or equal than current priority."""
- ...
-
- def __repr__(self) -> str:
- ...
-
-
-
-class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
- """
- Instances of this class behave like dictionaries, but store priorities
- along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
- immutable).
-
- Key-value entries can be passed on initialization with the ``values``
- argument, and they would take the ``priority`` level (unless ``values`` is
- already an instance of :class:`~scrapy.settings.BaseSettings`, in which
- case the existing priority levels will be kept). If the ``priority``
- argument is a string, the priority name will be looked up in
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
- should be provided.
-
- Once the object is created, new settings can be loaded or updated with the
- :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with
- the square bracket notation of dictionaries, or with the
- :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its
- value conversion variants. When requesting a stored key, the value with the
- highest priority will be retrieved.
- """
- __default = ...
- def __init__(self, values: _SettingsInputT = ..., priority: Union[int, str] = ...) -> None:
- ...
-
- def __getitem__(self, opt_name: _SettingsKeyT) -> Any:
- ...
-
- def __contains__(self, name: Any) -> bool:
- ...
-
- def get(self, name: _SettingsKeyT, default: Any = ...) -> Any:
- """
- Get a setting value without affecting its original type.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getbool(self, name: _SettingsKeyT, default: bool = ...) -> bool:
- """
- Get a setting value as a boolean.
-
- ``1``, ``'1'``, `True`` and ``'True'`` return ``True``,
- while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``.
-
- For example, settings populated through environment variables set to
- ``'0'`` will return ``False`` when using this method.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getint(self, name: _SettingsKeyT, default: int = ...) -> int:
- """
- Get a setting value as an int.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getfloat(self, name: _SettingsKeyT, default: float = ...) -> float:
- """
- Get a setting value as a float.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getlist(self, name: _SettingsKeyT, default: Optional[List[Any]] = ...) -> List[Any]:
- """
- Get a setting value as a list. If the setting original type is a list, a
- copy of it will be returned. If it's a string it will be split by ",".
-
- For example, settings populated through environment variables set to
- ``'one,two'`` will return a list ['one', 'two'] when using this method.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getdict(self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = ...) -> Dict[Any, Any]:
- """
- Get a setting value as a dictionary. If the setting original type is a
- dictionary, a copy of it will be returned. If it is a string it will be
- evaluated as a JSON dictionary. In the case that it is a
- :class:`~scrapy.settings.BaseSettings` instance itself, it will be
- converted to a dictionary, containing all its current settings values
- as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
- and losing all information about priority and mutability.
-
- :param name: the setting name
- :type name: str
-
- :param default: the value to return if no setting is found
- :type default: object
- """
- ...
-
- def getdictorlist(self, name: _SettingsKeyT, default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = ...) -> Union[Dict[Any, Any], List[Any]]:
- """Get a setting value as either a :class:`dict` or a :class:`list`.
-
- If the setting is already a dict or a list, a copy of it will be
- returned.
-
- If it is a string it will be evaluated as JSON, or as a comma-separated
- list of strings as a fallback.
-
- For example, settings populated from the command line will return:
-
- - ``{'key1': 'value1', 'key2': 'value2'}`` if set to
- ``'{"key1": "value1", "key2": "value2"}'``
-
- - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
-
- :param name: the setting name
- :type name: string
-
- :param default: the value to return if no setting is found
- :type default: any
- """
- ...
-
- def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
- """Get a composition of a dictionary-like setting and its `_BASE`
- counterpart.
-
- :param name: name of the dictionary-like setting
- :type name: str
- """
- ...
-
- def getpriority(self, name: _SettingsKeyT) -> Optional[int]:
- """
- Return the current numerical priority value of a setting, or ``None`` if
- the given ``name`` does not exist.
-
- :param name: the setting name
- :type name: str
- """
- ...
-
- def maxpriority(self) -> int:
- """
- Return the numerical value of the highest priority present throughout
- all settings, or the numerical value for ``default`` from
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings
- stored.
- """
- ...
-
- def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:
- ...
-
- def set(self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = ...) -> None:
- """
- Store a key/value attribute with a given priority.
-
- Settings should be populated *before* configuring the Crawler object
- (through the :meth:`~scrapy.crawler.Crawler.configure` method),
- otherwise they won't have any effect.
-
- :param name: the setting name
- :type name: str
-
- :param value: the value to associate with the setting
- :type value: object
-
- :param priority: the priority of the setting. Should be a key of
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
- :type priority: str or int
- """
- ...
-
- def setdefault(self, name: _SettingsKeyT, default: Any = ..., priority: Union[int, str] = ...) -> Any:
- ...
-
- def setdict(self, values: _SettingsInputT, priority: Union[int, str] = ...) -> None:
- ...
-
- def setmodule(self, module: Union[ModuleType, str], priority: Union[int, str] = ...) -> None:
- """
- Store settings from a module with a given priority.
-
- This is a helper function that calls
- :meth:`~scrapy.settings.BaseSettings.set` for every globally declared
- uppercase variable of ``module`` with the provided ``priority``.
-
- :param module: the module or the path of the module
- :type module: types.ModuleType or str
-
- :param priority: the priority of the settings. Should be a key of
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
- :type priority: str or int
- """
- ...
-
- def update(self, values: _SettingsInputT, priority: Union[int, str] = ...) -> None:
- """
- Store key/value pairs with a given priority.
-
- This is a helper function that calls
- :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values``
- with the provided ``priority``.
-
- If ``values`` is a string, it is assumed to be JSON-encoded and parsed
- into a dict with ``json.loads()`` first. If it is a
- :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities
- will be used and the ``priority`` parameter ignored. This allows
- inserting/updating settings with different priorities with a single
- command.
-
- :param values: the settings names and values
- :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
-
- :param priority: the priority of the settings. Should be a key of
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
- :type priority: str or int
- """
- ...
-
- def delete(self, name: _SettingsKeyT, priority: Union[int, str] = ...) -> None:
- ...
-
- def __delitem__(self, name: _SettingsKeyT) -> None:
- ...
-
- def copy(self) -> Self:
- """
- Make a deep copy of current settings.
-
- This method returns a new instance of the :class:`Settings` class,
- populated with the same values and their priorities.
-
- Modifications to the new object won't be reflected on the original
- settings.
- """
- ...
-
- def freeze(self) -> None:
- """
- Disable further changes to the current settings.
-
- After calling this method, the present state of the settings will become
- immutable. Trying to change values through the :meth:`~set` method and
- its variants won't be possible and will be alerted.
- """
- ...
-
- def frozencopy(self) -> Self:
- """
- Return an immutable copy of the current settings.
-
- Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`.
- """
- ...
-
- def __iter__(self) -> Iterator[_SettingsKeyT]:
- ...
-
- def __len__(self) -> int:
- ...
-
- def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]:
- """
- Make a copy of current settings and convert to a dict.
-
- This method returns a new dict populated with the same values
- and their priorities as the current settings.
-
- Modifications to the returned dict won't be reflected on the original
- settings.
-
- This method can be useful for example for printing settings
- in Scrapy shell.
- """
- ...
-
- def pop(self, name: _SettingsKeyT, default: Any = ...) -> Any:
- ...
-
-
-
-class Settings(BaseSettings):
- """
- This object stores Scrapy settings for the configuration of internal
- components, and can be used for any further customization.
-
- It is a direct subclass and supports all methods of
- :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation
- of this class, the new object will have the global default settings
- described on :ref:`topics-settings-ref` already populated.
- """
- def __init__(self, values: _SettingsInputT = ..., priority: Union[int, str] = ...) -> None:
- ...
-
-
-
-def iter_default_settings() -> Iterable[Tuple[str, Any]]:
- """Return the default settings as an iterator of (name, value) tuples"""
- ...
-
-def overridden_settings(settings: Mapping[_SettingsKeyT, Any]) -> Iterable[Tuple[str, Any]]:
- """Return an iterable of the settings that have been overridden"""
- ...
-
diff --git a/typings/scrapy/settings/default_settings.pyi b/typings/scrapy/settings/default_settings.pyi
deleted file mode 100644
index 3f7997f7..00000000
--- a/typings/scrapy/settings/default_settings.pyi
+++ /dev/null
@@ -1,184 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import sys
-
-"""
-This module contains the default values for all settings used by Scrapy.
-
-For more information about these settings you can read the settings
-documentation in docs/topics/settings.rst
-
-Scrapy developers, if you add a setting here remember to:
-
-* add it in alphabetical order
-* group similar settings without leaving blank lines
-* add its documentation to the available settings documentation
- (docs/topics/settings.rst)
-
-"""
-ADDONS = ...
-AJAXCRAWL_ENABLED = ...
-ASYNCIO_EVENT_LOOP = ...
-AUTOTHROTTLE_ENABLED = ...
-AUTOTHROTTLE_DEBUG = ...
-AUTOTHROTTLE_MAX_DELAY = ...
-AUTOTHROTTLE_START_DELAY = ...
-AUTOTHROTTLE_TARGET_CONCURRENCY = ...
-BOT_NAME = ...
-CLOSESPIDER_TIMEOUT = ...
-CLOSESPIDER_PAGECOUNT = ...
-CLOSESPIDER_ITEMCOUNT = ...
-CLOSESPIDER_ERRORCOUNT = ...
-COMMANDS_MODULE = ...
-COMPRESSION_ENABLED = ...
-CONCURRENT_ITEMS = ...
-CONCURRENT_REQUESTS = ...
-CONCURRENT_REQUESTS_PER_DOMAIN = ...
-CONCURRENT_REQUESTS_PER_IP = ...
-COOKIES_ENABLED = ...
-COOKIES_DEBUG = ...
-DEFAULT_ITEM_CLASS = ...
-DEFAULT_REQUEST_HEADERS = ...
-DEPTH_LIMIT = ...
-DEPTH_STATS_VERBOSE = ...
-DEPTH_PRIORITY = ...
-DNSCACHE_ENABLED = ...
-DNSCACHE_SIZE = ...
-DNS_RESOLVER = ...
-DNS_TIMEOUT = ...
-DOWNLOAD_DELAY = ...
-DOWNLOAD_HANDLERS = ...
-DOWNLOAD_HANDLERS_BASE = ...
-DOWNLOAD_TIMEOUT = ...
-DOWNLOAD_MAXSIZE = ...
-DOWNLOAD_WARNSIZE = ...
-DOWNLOAD_FAIL_ON_DATALOSS = ...
-DOWNLOADER = ...
-DOWNLOADER_HTTPCLIENTFACTORY = ...
-DOWNLOADER_CLIENTCONTEXTFACTORY = ...
-DOWNLOADER_CLIENT_TLS_CIPHERS = ...
-DOWNLOADER_CLIENT_TLS_METHOD = ...
-DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = ...
-DOWNLOADER_MIDDLEWARES = ...
-DOWNLOADER_MIDDLEWARES_BASE = ...
-DOWNLOADER_STATS = ...
-DUPEFILTER_CLASS = ...
-EDITOR = ...
-if sys.platform == "win32":
- ...
-EXTENSIONS = ...
-EXTENSIONS_BASE = ...
-FEED_TEMPDIR = ...
-FEEDS = ...
-FEED_URI_PARAMS = ...
-FEED_STORE_EMPTY = ...
-FEED_EXPORT_ENCODING = ...
-FEED_EXPORT_FIELDS = ...
-FEED_STORAGES = ...
-FEED_STORAGES_BASE = ...
-FEED_EXPORT_BATCH_ITEM_COUNT = ...
-FEED_EXPORTERS = ...
-FEED_EXPORTERS_BASE = ...
-FEED_EXPORT_INDENT = ...
-FEED_STORAGE_FTP_ACTIVE = ...
-FEED_STORAGE_GCS_ACL = ...
-FEED_STORAGE_S3_ACL = ...
-FILES_STORE_S3_ACL = ...
-FILES_STORE_GCS_ACL = ...
-FTP_USER = ...
-FTP_PASSWORD = ...
-FTP_PASSIVE_MODE = ...
-GCS_PROJECT_ID = ...
-HTTPCACHE_ENABLED = ...
-HTTPCACHE_DIR = ...
-HTTPCACHE_IGNORE_MISSING = ...
-HTTPCACHE_STORAGE = ...
-HTTPCACHE_EXPIRATION_SECS = ...
-HTTPCACHE_ALWAYS_STORE = ...
-HTTPCACHE_IGNORE_HTTP_CODES = ...
-HTTPCACHE_IGNORE_SCHEMES = ...
-HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = ...
-HTTPCACHE_DBM_MODULE = ...
-HTTPCACHE_POLICY = ...
-HTTPCACHE_GZIP = ...
-HTTPPROXY_ENABLED = ...
-HTTPPROXY_AUTH_ENCODING = ...
-IMAGES_STORE_S3_ACL = ...
-IMAGES_STORE_GCS_ACL = ...
-ITEM_PROCESSOR = ...
-ITEM_PIPELINES = ...
-ITEM_PIPELINES_BASE = ...
-LOG_ENABLED = ...
-LOG_ENCODING = ...
-LOG_FORMATTER = ...
-LOG_FORMAT = ...
-LOG_DATEFORMAT = ...
-LOG_STDOUT = ...
-LOG_LEVEL = ...
-LOG_FILE = ...
-LOG_FILE_APPEND = ...
-LOG_SHORT_NAMES = ...
-SCHEDULER_DEBUG = ...
-LOGSTATS_INTERVAL = ...
-MAIL_HOST = ...
-MAIL_PORT = ...
-MAIL_FROM = ...
-MAIL_PASS = ...
-MAIL_USER = ...
-MEMDEBUG_ENABLED = ...
-MEMDEBUG_NOTIFY = ...
-MEMUSAGE_CHECK_INTERVAL_SECONDS = ...
-MEMUSAGE_ENABLED = ...
-MEMUSAGE_LIMIT_MB = ...
-MEMUSAGE_NOTIFY_MAIL = ...
-MEMUSAGE_WARNING_MB = ...
-METAREFRESH_ENABLED = ...
-METAREFRESH_IGNORE_TAGS = ...
-METAREFRESH_MAXDELAY = ...
-NEWSPIDER_MODULE = ...
-PERIODIC_LOG_DELTA = ...
-PERIODIC_LOG_STATS = ...
-PERIODIC_LOG_TIMING_ENABLED = ...
-RANDOMIZE_DOWNLOAD_DELAY = ...
-REACTOR_THREADPOOL_MAXSIZE = ...
-REDIRECT_ENABLED = ...
-REDIRECT_MAX_TIMES = ...
-REDIRECT_PRIORITY_ADJUST = ...
-REFERER_ENABLED = ...
-REFERRER_POLICY = ...
-REQUEST_FINGERPRINTER_CLASS = ...
-REQUEST_FINGERPRINTER_IMPLEMENTATION = ...
-RETRY_ENABLED = ...
-RETRY_TIMES = ...
-RETRY_HTTP_CODES = ...
-RETRY_PRIORITY_ADJUST = ...
-RETRY_EXCEPTIONS = ...
-ROBOTSTXT_OBEY = ...
-ROBOTSTXT_PARSER = ...
-ROBOTSTXT_USER_AGENT = ...
-SCHEDULER = ...
-SCHEDULER_DISK_QUEUE = ...
-SCHEDULER_MEMORY_QUEUE = ...
-SCHEDULER_PRIORITY_QUEUE = ...
-SCRAPER_SLOT_MAX_ACTIVE_SIZE = ...
-SPIDER_LOADER_CLASS = ...
-SPIDER_LOADER_WARN_ONLY = ...
-SPIDER_MIDDLEWARES = ...
-SPIDER_MIDDLEWARES_BASE = ...
-SPIDER_MODULES = ...
-STATS_CLASS = ...
-STATS_DUMP = ...
-STATSMAILER_RCPTS = ...
-TEMPLATES_DIR = ...
-URLLENGTH_LIMIT = ...
-USER_AGENT = ...
-TELNETCONSOLE_ENABLED = ...
-TELNETCONSOLE_PORT = ...
-TELNETCONSOLE_HOST = ...
-TELNETCONSOLE_USERNAME = ...
-TELNETCONSOLE_PASSWORD = ...
-TWISTED_REACTOR = ...
-SPIDER_CONTRACTS = ...
-SPIDER_CONTRACTS_BASE = ...
diff --git a/typings/scrapy/shell.pyi b/typings/scrapy/shell.pyi
deleted file mode 100644
index e6641b70..00000000
--- a/typings/scrapy/shell.pyi
+++ /dev/null
@@ -1,35 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""Scrapy Shell
-
-See documentation in docs/topics/shell.rst
-
-"""
-class Shell:
- relevant_classes = ...
- def __init__(self, crawler, update_vars=..., code=...) -> None:
- ...
-
- def start(self, url=..., request=..., response=..., spider=..., redirect=...): # -> None:
- ...
-
- def fetch(self, request_or_url, spider=..., redirect=..., **kwargs): # -> None:
- ...
-
- def populate_vars(self, response=..., request=..., spider=...): # -> None:
- ...
-
- def print_help(self): # -> None:
- ...
-
- def get_help(self): # -> str:
- ...
-
-
-
-def inspect_response(response, spider): # -> None:
- """Open a shell to inspect the given response"""
- ...
-
diff --git a/typings/scrapy/signalmanager.pyi b/typings/scrapy/signalmanager.pyi
deleted file mode 100644
index 62c0c85d..00000000
--- a/typings/scrapy/signalmanager.pyi
+++ /dev/null
@@ -1,68 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, List, Tuple
-from twisted.internet.defer import Deferred
-
-class SignalManager:
- def __init__(self, sender: Any = ...) -> None:
- ...
-
- def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
- """
- Connect a receiver function to a signal.
-
- The signal can be any object, although Scrapy comes with some
- predefined signals that are documented in the :ref:`topics-signals`
- section.
-
- :param receiver: the function to be connected
- :type receiver: collections.abc.Callable
-
- :param signal: the signal to connect to
- :type signal: object
- """
- ...
-
- def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
- """
- Disconnect a receiver function from a signal. This has the
- opposite effect of the :meth:`connect` method, and the arguments
- are the same.
- """
- ...
-
- def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]:
- """
- Send a signal, catch exceptions and log them.
-
- The keyword arguments are passed to the signal handlers (connected
- through the :meth:`connect` method).
- """
- ...
-
- def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
- """
- Like :meth:`send_catch_log` but supports returning
- :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
-
- Returns a Deferred that gets fired once all signal handlers
- deferreds were fired. Send a signal, catch exceptions and log them.
-
- The keyword arguments are passed to the signal handlers (connected
- through the :meth:`connect` method).
- """
- ...
-
- def disconnect_all(self, signal: Any, **kwargs: Any) -> None:
- """
- Disconnect all receivers from the given signal.
-
- :param signal: the signal to disconnect from
- :type signal: object
- """
- ...
-
-
-
diff --git a/typings/scrapy/signals.pyi b/typings/scrapy/signals.pyi
deleted file mode 100644
index 434d998e..00000000
--- a/typings/scrapy/signals.pyi
+++ /dev/null
@@ -1,34 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Scrapy signals
-
-These signals are documented in docs/topics/signals.rst. Please don't add new
-signals here without documenting them there.
-"""
-engine_started = ...
-engine_stopped = ...
-spider_opened = ...
-spider_idle = ...
-spider_closed = ...
-spider_error = ...
-request_scheduled = ...
-request_dropped = ...
-request_reached_downloader = ...
-request_left_downloader = ...
-response_received = ...
-response_downloaded = ...
-headers_received = ...
-bytes_received = ...
-item_scraped = ...
-item_dropped = ...
-item_error = ...
-feed_slot_closed = ...
-feed_exporter_closed = ...
-stats_spider_opened = ...
-stats_spider_closing = ...
-stats_spider_closed = ...
-item_passed = ...
-request_received = ...
diff --git a/typings/scrapy/spiderloader.pyi b/typings/scrapy/spiderloader.pyi
deleted file mode 100644
index 8e5f27fc..00000000
--- a/typings/scrapy/spiderloader.pyi
+++ /dev/null
@@ -1,47 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import List, TYPE_CHECKING, Type
-from zope.interface import implementer
-from scrapy import Request, Spider
-from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import BaseSettings
-from typing_extensions import Self
-
-if TYPE_CHECKING:
- ...
-@implementer(ISpiderLoader)
-class SpiderLoader:
- """
- SpiderLoader is a class which locates and loads spiders
- in a Scrapy project.
- """
- def __init__(self, settings: BaseSettings) -> None:
- ...
-
- @classmethod
- def from_settings(cls, settings: BaseSettings) -> Self:
- ...
-
- def load(self, spider_name: str) -> Type[Spider]:
- """
- Return the Spider class for the given spider name. If the spider
- name is not found, raise a KeyError.
- """
- ...
-
- def find_by_request(self, request: Request) -> List[str]:
- """
- Return the list of spider names that can handle the given request.
- """
- ...
-
- def list(self) -> List[str]:
- """
- Return a list with the names of all spiders available in the project.
- """
- ...
-
-
-
diff --git a/typings/scrapy/spidermiddlewares/__init__.pyi b/typings/scrapy/spidermiddlewares/__init__.pyi
deleted file mode 100644
index 006bc274..00000000
--- a/typings/scrapy/spidermiddlewares/__init__.pyi
+++ /dev/null
@@ -1,4 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
diff --git a/typings/scrapy/spiders/__init__.pyi b/typings/scrapy/spiders/__init__.pyi
deleted file mode 100644
index 9c285ab9..00000000
--- a/typings/scrapy/spiders/__init__.pyi
+++ /dev/null
@@ -1,75 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import logging
-from __future__ import annotations
-from typing import Any, Iterable, List, Optional, TYPE_CHECKING, Union, cast
-from twisted.internet.defer import Deferred
-from scrapy import signals
-from scrapy.http import Request, Response
-from scrapy.utils.trackref import object_ref
-from scrapy.utils.url import url_is_from_spider
-from typing_extensions import Self
-from scrapy.crawler import Crawler
-from scrapy.settings import BaseSettings
-from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider
-from scrapy.spiders.sitemap import SitemapSpider
-
-"""
-Base class for Scrapy spiders
-
-See documentation in docs/topics/spiders.rst
-"""
-if TYPE_CHECKING:
- ...
-class Spider(object_ref):
- """Base class for scrapy spiders. All spiders must inherit from this
- class.
- """
- name: str
- custom_settings: Optional[dict] = ...
- def __init__(self, name: Optional[str] = ..., **kwargs: Any) -> None:
- ...
-
- @property
- def logger(self) -> logging.LoggerAdapter:
- ...
-
- def log(self, message: Any, level: int = ..., **kw: Any) -> None:
- """Log the given message at the given log level
-
- This helper wraps a log call to the logger within the spider, but you
- can use it directly (e.g. Spider.logger.info('msg')) or use any other
- Python logger too.
- """
- ...
-
- @classmethod
- def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
- ...
-
- def start_requests(self) -> Iterable[Request]:
- ...
-
- def parse(self, response: Response, **kwargs: Any) -> Any:
- ...
-
- @classmethod
- def update_settings(cls, settings: BaseSettings) -> None:
- ...
-
- @classmethod
- def handles_request(cls, request: Request) -> bool:
- ...
-
- @staticmethod
- def close(spider: Spider, reason: str) -> Union[Deferred, None]:
- ...
-
- def __repr__(self) -> str:
- ...
-
-
-
diff --git a/typings/scrapy/spiders/crawl.pyi b/typings/scrapy/spiders/crawl.pyi
deleted file mode 100644
index cfbcd688..00000000
--- a/typings/scrapy/spiders/crawl.pyi
+++ /dev/null
@@ -1,38 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Sequence
-from scrapy.http import Response
-from scrapy.spiders import Spider
-
-"""
-This modules implements the CrawlSpider which is the recommended spider to use
-for scraping typical web sites that requires crawling pages.
-
-See documentation in docs/topics/spiders.rst
-"""
-_default_link_extractor = ...
-class Rule:
- def __init__(self, link_extractor=..., callback=..., cb_kwargs=..., follow=..., process_links=..., process_request=..., errback=...) -> None:
- ...
-
-
-
-class CrawlSpider(Spider):
- rules: Sequence[Rule] = ...
- def __init__(self, *a, **kw) -> None:
- ...
-
- def parse_start_url(self, response, **kwargs): # -> list[Any]:
- ...
-
- def process_results(self, response: Response, results: list): # -> list[Any]:
- ...
-
- @classmethod
- def from_crawler(cls, crawler, *args, **kwargs): # -> Self:
- ...
-
-
-
diff --git a/typings/scrapy/spiders/feed.pyi b/typings/scrapy/spiders/feed.pyi
deleted file mode 100644
index 132fc3b8..00000000
--- a/typings/scrapy/spiders/feed.pyi
+++ /dev/null
@@ -1,89 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.spiders import Spider
-
-"""
-This module implements the XMLFeedSpider which is the recommended spider to use
-for scraping from an XML feed.
-
-See documentation in docs/topics/spiders.rst
-"""
-class XMLFeedSpider(Spider):
- """
- This class intends to be the base class for spiders that scrape
- from XML feeds.
-
- You can choose whether to parse the file using the 'iternodes' iterator, an
- 'xml' selector, or an 'html' selector. In most cases, it's convenient to
- use iternodes, since it's a faster and cleaner.
- """
- iterator = ...
- itertag = ...
- namespaces = ...
- def process_results(self, response, results):
- """This overridable method is called for each result (item or request)
- returned by the spider, and it's intended to perform any last time
- processing required before returning the results to the framework core,
- for example setting the item GUIDs. It receives a list of results and
- the response which originated that results. It must return a list of
- results (items or requests).
- """
- ...
-
- def adapt_response(self, response):
- """You can override this function in order to make any changes you want
- to into the feed before parsing it. This function must return a
- response.
- """
- ...
-
- def parse_node(self, response, selector):
- """This method must be overridden with your custom spider functionality"""
- ...
-
- def parse_nodes(self, response, nodes): # -> Generator[Any, Any, None]:
- """This method is called for the nodes matching the provided tag name
- (itertag). Receives the response and an Selector for each node.
- Overriding this method is mandatory. Otherwise, you spider won't work.
- This method must return either an item, a request, or a list
- containing any of them.
- """
- ...
-
-
-
-class CSVFeedSpider(Spider):
- """Spider for parsing CSV feeds.
- It receives a CSV file in a response; iterates through each of its rows,
- and calls parse_row with a dict containing each field's data.
-
- You can set some options regarding the CSV file, such as the delimiter, quotechar
- and the file's headers.
- """
- delimiter = ...
- quotechar = ...
- headers = ...
- def process_results(self, response, results):
- """This method has the same purpose as the one in XMLFeedSpider"""
- ...
-
- def adapt_response(self, response):
- """This method has the same purpose as the one in XMLFeedSpider"""
- ...
-
- def parse_row(self, response, row):
- """This method must be overridden with your custom spider functionality"""
- ...
-
- def parse_rows(self, response): # -> Generator[Any, Any, None]:
- """Receives a response and a dict (representing each row) with a key for
- each provided (or detected) header of the CSV file. This spider also
- gives the opportunity to override adapt_response and
- process_results methods for pre and post-processing purposes.
- """
- ...
-
-
-
diff --git a/typings/scrapy/spiders/sitemap.pyi b/typings/scrapy/spiders/sitemap.pyi
deleted file mode 100644
index b6f73541..00000000
--- a/typings/scrapy/spiders/sitemap.pyi
+++ /dev/null
@@ -1,33 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.spiders import Spider
-
-logger = ...
-class SitemapSpider(Spider):
- sitemap_urls = ...
- sitemap_rules = ...
- sitemap_follow = ...
- sitemap_alternate_links = ...
- def __init__(self, *a, **kw) -> None:
- ...
-
- def start_requests(self): # -> Generator[Request, Any, None]:
- ...
-
- def sitemap_filter(self, entries): # -> Generator[Any, Any, None]:
- """This method can be used to filter sitemap entries by their
- attributes, for example, you can filter locs with lastmod greater
- than a given date (see docs).
- """
- ...
-
-
-
-def regex(x): # -> Pattern[str]:
- ...
-
-def iterloc(it, alt=...): # -> Generator[Any, Any, None]:
- ...
-
diff --git a/typings/scrapy/squeues.pyi b/typings/scrapy/squeues.pyi
deleted file mode 100644
index 88a8257a..00000000
--- a/typings/scrapy/squeues.pyi
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-Scheduler queues
-"""
-_PickleFifoSerializationDiskQueue = ...
-_PickleLifoSerializationDiskQueue = ...
-_MarshalFifoSerializationDiskQueue = ...
-_MarshalLifoSerializationDiskQueue = ...
-PickleFifoDiskQueue = ...
-PickleLifoDiskQueue = ...
-MarshalFifoDiskQueue = ...
-MarshalLifoDiskQueue = ...
-FifoMemoryQueue = ...
-LifoMemoryQueue = ...
diff --git a/typings/scrapy/statscollectors.pyi b/typings/scrapy/statscollectors.pyi
deleted file mode 100644
index d3b8b1b4..00000000
--- a/typings/scrapy/statscollectors.pyi
+++ /dev/null
@@ -1,78 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Dict, Optional, TYPE_CHECKING
-from scrapy import Spider
-from scrapy.crawler import Crawler
-
-"""
-Scrapy extension for collecting scraping stats
-"""
-if TYPE_CHECKING:
- ...
-logger = ...
-StatsT = Dict[str, Any]
-class StatsCollector:
- def __init__(self, crawler: Crawler) -> None:
- ...
-
- def get_value(self, key: str, default: Any = ..., spider: Optional[Spider] = ...) -> Any:
- ...
-
- def get_stats(self, spider: Optional[Spider] = ...) -> StatsT:
- ...
-
- def set_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
- def set_stats(self, stats: StatsT, spider: Optional[Spider] = ...) -> None:
- ...
-
- def inc_value(self, key: str, count: int = ..., start: int = ..., spider: Optional[Spider] = ...) -> None:
- ...
-
- def max_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
- def min_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
- def clear_stats(self, spider: Optional[Spider] = ...) -> None:
- ...
-
- def open_spider(self, spider: Spider) -> None:
- ...
-
- def close_spider(self, spider: Spider, reason: str) -> None:
- ...
-
-
-
-class MemoryStatsCollector(StatsCollector):
- def __init__(self, crawler: Crawler) -> None:
- ...
-
-
-
-class DummyStatsCollector(StatsCollector):
- def get_value(self, key: str, default: Any = ..., spider: Optional[Spider] = ...) -> Any:
- ...
-
- def set_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
- def set_stats(self, stats: StatsT, spider: Optional[Spider] = ...) -> None:
- ...
-
- def inc_value(self, key: str, count: int = ..., start: int = ..., spider: Optional[Spider] = ...) -> None:
- ...
-
- def max_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
- def min_value(self, key: str, value: Any, spider: Optional[Spider] = ...) -> None:
- ...
-
-
-
diff --git a/typings/scrapy/utils/__init__.pyi b/typings/scrapy/utils/__init__.pyi
deleted file mode 100644
index 006bc274..00000000
--- a/typings/scrapy/utils/__init__.pyi
+++ /dev/null
@@ -1,4 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
diff --git a/typings/scrapy/utils/asyncgen.pyi b/typings/scrapy/utils/asyncgen.pyi
deleted file mode 100644
index 87f3005e..00000000
--- a/typings/scrapy/utils/asyncgen.pyi
+++ /dev/null
@@ -1,13 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import AsyncGenerator, AsyncIterable, Iterable, Union
-
-async def collect_asyncgen(result: AsyncIterable) -> list:
- ...
-
-async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
- """Wraps an iterable (sync or async) into an async generator."""
- ...
-
diff --git a/typings/scrapy/utils/conf.pyi b/typings/scrapy/utils/conf.pyi
deleted file mode 100644
index a1921767..00000000
--- a/typings/scrapy/utils/conf.pyi
+++ /dev/null
@@ -1,50 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import os
-from configparser import ConfigParser
-from typing import Any, Callable, Dict, List, MutableMapping, Optional, Union
-from scrapy.settings import BaseSettings
-
-def build_component_list(compdict: MutableMapping[Any, Any], custom: Any = ..., convert: Callable[[Any], Any] = ...) -> List[Any]:
- """Compose a component list from a { class: order } dictionary."""
- ...
-
-def arglist_to_dict(arglist: List[str]) -> Dict[str, str]:
- """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
- dict
- """
- ...
-
-def closest_scrapy_cfg(path: Union[str, os.PathLike] = ..., prevpath: Optional[Union[str, os.PathLike]] = ...) -> str:
- """Return the path to the closest scrapy.cfg file by traversing the current
- directory and its parents
- """
- ...
-
-def init_env(project: str = ..., set_syspath: bool = ...) -> None:
- """Initialize environment to use command-line tool from inside a project
- dir. This sets the Scrapy settings module and modifies the Python path to
- be able to locate the project module.
- """
- ...
-
-def get_config(use_closest: bool = ...) -> ConfigParser:
- """Get Scrapy config file as a ConfigParser"""
- ...
-
-def get_sources(use_closest: bool = ...) -> List[str]:
- ...
-
-def feed_complete_default_values_from_settings(feed: Dict[str, Any], settings: BaseSettings) -> Dict[str, Any]:
- ...
-
-def feed_process_params_from_cli(settings: BaseSettings, output: List[str], output_format: Optional[str] = ..., overwrite_output: Optional[List[str]] = ...) -> Dict[str, Dict[str, Any]]:
- """
- Receives feed export params (from the 'crawl' or 'runspider' commands),
- checks for inconsistencies in their quantities and returns a dictionary
- suitable to be used as the FEEDS setting.
- """
- ...
-
diff --git a/typings/scrapy/utils/console.pyi b/typings/scrapy/utils/console.pyi
deleted file mode 100644
index 1ad7a39f..00000000
--- a/typings/scrapy/utils/console.pyi
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-DEFAULT_PYTHON_SHELLS = ...
-def get_shell_embed_func(shells=..., known_shells=...): # -> _Wrapped[Callable[..., Any], _Wrapped[Callable[..., Any], Any, Callable[..., Any], None], Callable[..., Any], None] | None:
- """Return the first acceptable shell-embed function
- from a given list of shell names.
- """
- ...
-
-def start_python_console(namespace=..., banner=..., shells=...): # -> None:
- """Start Python console bound to the given namespace.
- Readline support and tab completion will be used on Unix, if available.
- """
- ...
-
diff --git a/typings/scrapy/utils/curl.pyi b/typings/scrapy/utils/curl.pyi
deleted file mode 100644
index 7de5a891..00000000
--- a/typings/scrapy/utils/curl.pyi
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import argparse
-
-class DataAction(argparse.Action):
- def __call__(self, parser, namespace, values, option_string=...): # -> None:
- ...
-
-
-
-class CurlParser(argparse.ArgumentParser):
- def error(self, message):
- ...
-
-
-
-curl_parser = ...
-safe_to_ignore_arguments = ...
-def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = ...) -> dict:
- """Convert a cURL command syntax to Request kwargs.
-
- :param str curl_command: string containing the curl command
- :param bool ignore_unknown_options: If true, only a warning is emitted when
- cURL options are unknown. Otherwise
- raises an error. (default: True)
- :return: dictionary of Request kwargs
- """
- ...
-
diff --git a/typings/scrapy/utils/datatypes.pyi b/typings/scrapy/utils/datatypes.pyi
deleted file mode 100644
index 895b697b..00000000
--- a/typings/scrapy/utils/datatypes.pyi
+++ /dev/null
@@ -1,136 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import collections
-import weakref
-from typing import Any, AnyStr, Optional, OrderedDict, Sequence, TypeVar
-
-"""
-This module contains data types used by Scrapy which are not included in the
-Python Standard Library.
-
-This module must not depend on any module outside the Standard Library.
-"""
-_KT = TypeVar("_KT")
-_VT = TypeVar("_VT")
-class CaselessDict(dict):
- __slots__ = ...
- def __new__(cls, *args, **kwargs): # -> Self:
- ...
-
- def __init__(self, seq=...) -> None:
- ...
-
- def __getitem__(self, key):
- ...
-
- def __setitem__(self, key, value): # -> None:
- ...
-
- def __delitem__(self, key): # -> None:
- ...
-
- def __contains__(self, key): # -> bool:
- ...
-
- has_key = ...
- def __copy__(self): # -> Self:
- ...
-
- copy = ...
- def normkey(self, key):
- """Method to normalize dictionary key access"""
- ...
-
- def normvalue(self, value):
- """Method to normalize values prior to be set"""
- ...
-
- def get(self, key, def_val=...):
- ...
-
- def setdefault(self, key, def_val=...):
- ...
-
- def update(self, seq): # -> None:
- ...
-
- @classmethod
- def fromkeys(cls, keys, value=...): # -> Self:
- ...
-
- def pop(self, key, *args):
- ...
-
-
-
-class CaseInsensitiveDict(collections.UserDict):
- """A dict-like structure that accepts strings or bytes
- as keys and allows case-insensitive lookups.
- """
- def __init__(self, *args, **kwargs) -> None:
- ...
-
- def __getitem__(self, key: AnyStr) -> Any:
- ...
-
- def __setitem__(self, key: AnyStr, value: Any) -> None:
- ...
-
- def __delitem__(self, key: AnyStr) -> None:
- ...
-
- def __contains__(self, key: AnyStr) -> bool:
- ...
-
- def __repr__(self) -> str:
- ...
-
-
-
-class LocalCache(OrderedDict[_KT, _VT]):
- """Dictionary with a finite number of keys.
-
- Older items expires first.
- """
- def __init__(self, limit: Optional[int] = ...) -> None:
- ...
-
- def __setitem__(self, key: _KT, value: _VT) -> None:
- ...
-
-
-
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
- """
- A weakref.WeakKeyDictionary implementation that uses LocalCache as its
- underlying data structure, making it ordered and capable of being size-limited.
-
- Useful for memoization, while avoiding keeping received
- arguments in memory only because of the cached references.
-
- Note: like LocalCache and unlike weakref.WeakKeyDictionary,
- it cannot be instantiated with an initial dictionary.
- """
- def __init__(self, limit: Optional[int] = ...) -> None:
- ...
-
- def __setitem__(self, key: _KT, value: _VT) -> None:
- ...
-
- def __getitem__(self, key: _KT) -> Optional[_VT]:
- ...
-
-
-
-class SequenceExclude:
- """Object to test if an item is NOT within some sequence."""
- def __init__(self, seq: Sequence) -> None:
- ...
-
- def __contains__(self, item: Any) -> bool:
- ...
-
-
-
diff --git a/typings/scrapy/utils/decorators.pyi b/typings/scrapy/utils/decorators.pyi
deleted file mode 100644
index d3cba29c..00000000
--- a/typings/scrapy/utils/decorators.pyi
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Callable
-from twisted.internet.defer import Deferred
-
-def deprecated(use_instead: Any = ...) -> Callable:
- """This is a decorator which can be used to mark functions
- as deprecated. It will result in a warning being emitted
- when the function is used."""
- ...
-
-def defers(func: Callable) -> Callable[..., Deferred]:
- """Decorator to make sure a function always returns a deferred"""
- ...
-
-def inthread(func: Callable) -> Callable[..., Deferred]:
- """Decorator to call a function in a thread and return a deferred with the
- result
- """
- ...
-
diff --git a/typings/scrapy/utils/defer.pyi b/typings/scrapy/utils/defer.pyi
deleted file mode 100644
index 2d6bf403..00000000
--- a/typings/scrapy/utils/defer.pyi
+++ /dev/null
@@ -1,206 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from asyncio import Future
-from types import CoroutineType
-from typing import Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, Coroutine, Generator, Iterable, Iterator, TypeVar, Union, overload
-from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
-
-"""
-Helper functions for dealing with Twisted deferreds
-"""
-def defer_fail(_failure: Failure) -> Deferred:
- """Same as twisted.internet.defer.fail but delay calling errback until
- next reactor loop
-
- It delays by 100ms so reactor has a chance to go through readers and writers
- before attending pending delayed calls, so do not set delay to zero.
- """
- ...
-
-def defer_succeed(result: Any) -> Deferred:
- """Same as twisted.internet.defer.succeed but delay calling callback until
- next reactor loop
-
- It delays by 100ms so reactor has a chance to go through readers and writers
- before attending pending delayed calls, so do not set delay to zero.
- """
- ...
-
-def defer_result(result: Any) -> Deferred:
- ...
-
-def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred:
- """Same as twisted.internet.defer.maybeDeferred, but delay calling
- callback/errback to next reactor loop
- """
- ...
-
-def parallel(iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any) -> Deferred:
- """Execute a callable over the objects in the given iterable, in parallel,
- using no more than ``count`` concurrent calls.
-
- Taken from: https://jcalderone.livejournal.com/24285.html
- """
- ...
-
-class _AsyncCooperatorAdapter(Iterator):
- """A class that wraps an async iterable into a normal iterator suitable
- for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
- it calls the callable directly in the callback, instead of providing a more
- generic interface.
-
- On the outside, this class behaves as an iterator that yields Deferreds.
- Each Deferred is fired with the result of the callable which was called on
- the next result from aiterator. It raises StopIteration when aiterator is
- exhausted, as expected.
-
- Cooperator calls __next__() multiple times and waits on the Deferreds
- returned from it. As async generators (since Python 3.8) don't support
- awaiting on __anext__() several times in parallel, we need to serialize
- this. It's done by storing the Deferreds returned from __next__() and
- firing the oldest one when a result from __anext__() is available.
-
- The workflow:
- 1. When __next__() is called for the first time, it creates a Deferred, stores it
- in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
- for self.aiterator.__anext__() and puts it into self.anext_deferred.
- 2. If __next__() is called again before self.anext_deferred fires, more Deferreds
- are added to self.waiting_deferreds.
- 3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
- clear self.anext_deferred.
- 3.1. _callback() calls the callable passing the result value that it takes, pops a
- Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
- chains those Deferreds so that the waiting Deferred will fire when the result
- Deferred does, otherwise it fires it directly. This causes one awaiting task to
- receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
- called and self.anext_deferred is populated.
- 3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
- self.aiterator is exhausted and so it sets self.finished and fires all
- self.waiting_deferreds. Other exceptions are propagated.
- 4. If __next__() is called after __anext__() was handled, then if self.finished is
- True, it raises StopIteration, otherwise it acts like in step 2, but if
- self.anext_deferred is now empty is also populates it with a new __anext__().
-
- Note that CooperativeTask ignores the value returned from the Deferred that it waits
- for, so we fire them with None when needed.
-
- It may be possible to write an async iterator-aware replacement for
- Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
- goal.
- """
- def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args: Any, **callable_kwargs: Any) -> None:
- ...
-
- def __next__(self) -> Deferred:
- ...
-
-
-
-def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args: Any, **named: Any) -> Deferred:
- """Like parallel but for async iterators"""
- ...
-
-def process_chain(callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred:
- """Return a Deferred built by chaining the given callbacks"""
- ...
-
-def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred:
- """Return a Deferred built by chaining the given callbacks and errbacks"""
- ...
-
-def process_parallel(callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any) -> Deferred:
- """Return a Deferred with the output of all successful calls to the given
- callbacks
- """
- ...
-
-def iter_errback(iterable: Iterable, errback: Callable, *a: Any, **kw: Any) -> Generator:
- """Wraps an iterable calling an errback if an error is caught while
- iterating it.
- """
- ...
-
-async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any) -> AsyncGenerator:
- """Wraps an async iterable calling an errback if an error is caught while
- iterating it. Similar to scrapy.utils.defer.iter_errback()
- """
- ...
-
-_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
-_T = TypeVar("_T")
-@overload
-def deferred_from_coro(o: _CT) -> Deferred:
- ...
-
-@overload
-def deferred_from_coro(o: _T) -> _T:
- ...
-
-def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
- """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
- ...
-
-def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
- """Converts a coroutine function into a function that returns a Deferred.
-
- The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
- This is useful for callback chains, as callback functions are called with the previous callback result.
- """
- ...
-
-def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred:
- """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
- ...
-
-def deferred_to_future(d: Deferred) -> Future:
- """
- .. versionadded:: 2.6.0
-
- Return an :class:`asyncio.Future` object that wraps *d*.
-
- When :ref:`using the asyncio reactor `, you cannot await
- on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
- callables defined as coroutines `, you can only await on
- ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
- allows you to wait on them::
-
- class MySpider(Spider):
- ...
- async def parse(self, response):
- additional_request = scrapy.Request('https://example.org/price')
- deferred = self.crawler.engine.download(additional_request)
- additional_response = await deferred_to_future(deferred)
- """
- ...
-
-def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
- """
- .. versionadded:: 2.6.0
-
- Return *d* as an object that can be awaited from a :ref:`Scrapy callable
- defined as a coroutine `.
-
- What you can await in Scrapy callables defined as coroutines depends on the
- value of :setting:`TWISTED_REACTOR`:
-
- - When not using the asyncio reactor, you can only await on
- :class:`~twisted.internet.defer.Deferred` objects.
-
- - When :ref:`using the asyncio reactor `, you can only
- await on :class:`asyncio.Future` objects.
-
- If you want to write code that uses ``Deferred`` objects but works with any
- reactor, use this function on all ``Deferred`` objects::
-
- class MySpider(Spider):
- ...
- async def parse(self, response):
- additional_request = scrapy.Request('https://example.org/price')
- deferred = self.crawler.engine.download(additional_request)
- additional_response = await maybe_deferred_to_future(deferred)
- """
- ...
-
diff --git a/typings/scrapy/utils/deprecate.pyi b/typings/scrapy/utils/deprecate.pyi
deleted file mode 100644
index da38d02a..00000000
--- a/typings/scrapy/utils/deprecate.pyi
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Dict, List, Optional, Tuple, Type, overload
-
-"""Some helpers for deprecation messages"""
-def attribute(obj: Any, oldattr: str, newattr: str, version: str = ...) -> None:
- ...
-
-def create_deprecated_class(name: str, new_class: type, clsdict: Optional[Dict[str, Any]] = ..., warn_category: Type[Warning] = ..., warn_once: bool = ..., old_class_path: Optional[str] = ..., new_class_path: Optional[str] = ..., subclass_warn_message: str = ..., instance_warn_message: str = ...) -> type:
- """
- Return a "deprecated" class that causes its subclasses to issue a warning.
- Subclasses of ``new_class`` are considered subclasses of this class.
- It also warns when the deprecated class is instantiated, but do not when
- its subclasses are instantiated.
-
- It can be used to rename a base class in a library. For example, if we
- have
-
- class OldName(SomeClass):
- # ...
-
- and we want to rename it to NewName, we can do the following::
-
- class NewName(SomeClass):
- # ...
-
- OldName = create_deprecated_class('OldName', NewName)
-
- Then, if user class inherits from OldName, warning is issued. Also, if
- some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``
- checks they'll still return True if sub is a subclass of NewName instead of
- OldName.
- """
- class DeprecatedClass(new_class.__class__):
- ...
-
-
-
-DEPRECATION_RULES: List[Tuple[str, str]] = ...
-@overload
-def update_classpath(path: str) -> str:
- ...
-
-@overload
-def update_classpath(path: Any) -> Any:
- ...
-
-def update_classpath(path: Any) -> Any:
- """Update a deprecated path from an object with its new location"""
- ...
-
-def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool:
- """
- Return True if a method named ``method_name`` of a ``base_class``
- is overridden in a ``subclass``.
-
- >>> class Base:
- ... def foo(self):
- ... pass
- >>> class Sub1(Base):
- ... pass
- >>> class Sub2(Base):
- ... def foo(self):
- ... pass
- >>> class Sub3(Sub1):
- ... def foo(self):
- ... pass
- >>> class Sub4(Sub2):
- ... pass
- >>> method_is_overridden(Sub1, Base, 'foo')
- False
- >>> method_is_overridden(Sub2, Base, 'foo')
- True
- >>> method_is_overridden(Sub3, Base, 'foo')
- True
- >>> method_is_overridden(Sub4, Base, 'foo')
- True
- """
- ...
-
diff --git a/typings/scrapy/utils/gz.pyi b/typings/scrapy/utils/gz.pyi
deleted file mode 100644
index 1ba6bfcc..00000000
--- a/typings/scrapy/utils/gz.pyi
+++ /dev/null
@@ -1,16 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.http import Response
-
-def gunzip(data: bytes) -> bytes:
- """Gunzip the given data and return as much data as possible.
-
- This is resilient to CRC checksum errors.
- """
- ...
-
-def gzip_magic_number(response: Response) -> bool:
- ...
-
diff --git a/typings/scrapy/utils/httpobj.pyi b/typings/scrapy/utils/httpobj.pyi
deleted file mode 100644
index f7d2ac43..00000000
--- a/typings/scrapy/utils/httpobj.pyi
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Union
-from urllib.parse import ParseResult
-from weakref import WeakKeyDictionary
-from scrapy.http import Request, Response
-
-"""Helper functions for scrapy.http objects (Request, Response)"""
-_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ...
-def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
- """Return urlparse.urlparse caching the result, where the argument can be a
- Request or Response object
- """
- ...
-
diff --git a/typings/scrapy/utils/iterators.pyi b/typings/scrapy/utils/iterators.pyi
deleted file mode 100644
index 300eab76..00000000
--- a/typings/scrapy/utils/iterators.pyi
+++ /dev/null
@@ -1,51 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING, Union
-from scrapy.http import Response
-from scrapy.selector import Selector
-
-if TYPE_CHECKING:
- ...
-logger = ...
-def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Generator[Selector, Any, None]:
- """Return a iterator of Selector's over all nodes of a XML document,
- given the name of the node to iterate. Useful for parsing XML feeds.
-
- obj can be:
- - a Response object
- - a unicode string
- - a string encoded as utf-8
- """
- ...
-
-def xmliter_lxml(obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = ..., prefix: str = ...) -> Generator[Selector, Any, None]:
- ...
-
-class _StreamReader:
- def __init__(self, obj: Union[Response, str, bytes]) -> None:
- ...
-
- def read(self, n: int = ...) -> bytes:
- ...
-
-
-
-def csviter(obj: Union[Response, str, bytes], delimiter: Optional[str] = ..., headers: Optional[List[str]] = ..., encoding: Optional[str] = ..., quotechar: Optional[str] = ...) -> Generator[Dict[str, str], Any, None]:
- """Returns an iterator of dictionaries from the given csv object
-
- obj can be:
- - a Response object
- - a unicode string
- - a string encoded as utf-8
-
- delimiter is the character used to separate fields on the given obj.
-
- headers is an iterable that when provided offers the keys
- for the returned dictionaries, if not the first row is used.
-
- quotechar is the character used to enclosure fields on the given obj.
- """
- ...
-
diff --git a/typings/scrapy/utils/job.pyi b/typings/scrapy/utils/job.pyi
deleted file mode 100644
index 08ed3919..00000000
--- a/typings/scrapy/utils/job.pyi
+++ /dev/null
@@ -1,10 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Optional
-from scrapy.settings import BaseSettings
-
-def job_dir(settings: BaseSettings) -> Optional[str]:
- ...
-
diff --git a/typings/scrapy/utils/log.pyi b/typings/scrapy/utils/log.pyi
deleted file mode 100644
index 7198255c..00000000
--- a/typings/scrapy/utils/log.pyi
+++ /dev/null
@@ -1,112 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import logging
-from types import TracebackType
-from typing import Any, List, Optional, TYPE_CHECKING, Tuple, Type, Union
-from twisted.python.failure import Failure
-from scrapy.settings import Settings
-from scrapy.crawler import Crawler
-
-if TYPE_CHECKING:
- ...
-logger = ...
-def failure_to_exc_info(failure: Failure) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:
- """Extract exc_info from Failure instances"""
- ...
-
-class TopLevelFormatter(logging.Filter):
- """Keep only top level loggers's name (direct children from root) from
- records.
-
- This filter will replace Scrapy loggers' names with 'scrapy'. This mimics
- the old Scrapy log behaviour and helps shortening long names.
-
- Since it can't be set for just one logger (it won't propagate for its
- children), it's going to be set in the root handler, with a parametrized
- ``loggers`` list where it should act.
- """
- def __init__(self, loggers: Optional[List[str]] = ...) -> None:
- ...
-
- def filter(self, record: logging.LogRecord) -> bool:
- ...
-
-
-
-DEFAULT_LOGGING = ...
-def configure_logging(settings: Union[Settings, dict, None] = ..., install_root_handler: bool = ...) -> None:
- """
- Initialize logging defaults for Scrapy.
-
- :param settings: settings used to create and configure a handler for the
- root logger (default: None).
- :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
-
- :param install_root_handler: whether to install root logging handler
- (default: True)
- :type install_root_handler: bool
-
- This function does:
-
- - Route warnings and twisted logging through Python standard logging
- - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
- - Route stdout to log if LOG_STDOUT setting is True
-
- When ``install_root_handler`` is True (default), this function also
- creates a handler for the root logger according to given settings
- (see :ref:`topics-logging-settings`). You can override default options
- using ``settings`` argument. When ``settings`` is empty or None, defaults
- are used.
- """
- ...
-
-_scrapy_root_handler: Optional[logging.Handler] = ...
-def install_scrapy_root_handler(settings: Settings) -> None:
- ...
-
-def get_scrapy_root_handler() -> Optional[logging.Handler]:
- ...
-
-def log_scrapy_info(settings: Settings) -> None:
- ...
-
-def log_reactor_info() -> None:
- ...
-
-class StreamLogger:
- """Fake file-like stream object that redirects writes to a logger instance
-
- Taken from:
- https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
- """
- def __init__(self, logger: logging.Logger, log_level: int = ...) -> None:
- ...
-
- def write(self, buf: str) -> None:
- ...
-
- def flush(self) -> None:
- ...
-
-
-
-class LogCounterHandler(logging.Handler):
- """Record log levels count into a crawler stats"""
- def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any) -> None:
- ...
-
- def emit(self, record: logging.LogRecord) -> None:
- ...
-
-
-
-def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
- """
- Helper that takes the dictionary output from the methods in LogFormatter
- and adapts it into a tuple of positional arguments for logger.log calls,
- handling backward compatibility as well.
- """
- ...
-
diff --git a/typings/scrapy/utils/misc.pyi b/typings/scrapy/utils/misc.pyi
deleted file mode 100644
index 59d32c71..00000000
--- a/typings/scrapy/utils/misc.pyi
+++ /dev/null
@@ -1,112 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import ast
-from contextlib import contextmanager
-from types import ModuleType
-from typing import Any, Callable, Generator, IO, Iterable, List, Optional, Pattern, TYPE_CHECKING, Union
-from scrapy import Spider
-
-"""Helper functions which don't fit anywhere else"""
-if TYPE_CHECKING:
- ...
-_ITERABLE_SINGLE_VALUES = ...
-def arg_to_iter(arg: Any) -> Iterable[Any]:
- """Convert an argument to an iterable. The argument can be a None, single
- value, or an iterable.
-
- Exception: if arg is a dict, [arg] will be returned
- """
- ...
-
-def load_object(path: Union[str, Callable]) -> Any:
- """Load an object given its absolute object path, and return it.
-
- The object can be the import path of a class, function, variable or an
- instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.
-
- If ``path`` is not a string, but is a callable object, such as a class or
- a function, then return it as is.
- """
- ...
-
-def walk_modules(path: str) -> List[ModuleType]:
- """Loads a module and all its submodules from the given module path and
- returns them. If *any* module throws an exception while importing, that
- exception is thrown back.
-
- For example: walk_modules('scrapy.utils')
- """
- ...
-
-def extract_regex(regex: Union[str, Pattern], text: str, encoding: str = ...) -> List[str]:
- """Extract a list of unicode strings from the given text/encoding using the following policies:
-
- * if the regex contains a named group called "extract" that will be returned
- * if the regex contains multiple numbered groups, all those will be returned (flattened)
- * if the regex doesn't contain any group the entire regex matching is returned
- """
- ...
-
-def md5sum(file: IO) -> str:
- """Calculate the md5 checksum of a file-like object without reading its
- whole content in memory.
-
- >>> from io import BytesIO
- >>> md5sum(BytesIO(b'file content to hash'))
- '784406af91dd5a54fbb9c84c2236595a'
- """
- ...
-
-def rel_has_nofollow(rel: Optional[str]) -> bool:
- """Return True if link rel attribute has nofollow type"""
- ...
-
-def create_instance(objcls, settings, crawler, *args, **kwargs):
- """Construct a class instance using its ``from_crawler`` or
- ``from_settings`` constructors, if available.
-
- At least one of ``settings`` and ``crawler`` needs to be different from
- ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
- If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
- tried.
-
- ``*args`` and ``**kwargs`` are forwarded to the constructors.
-
- Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
-
- .. versionchanged:: 2.2
- Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
- extension has not been implemented correctly).
- """
- ...
-
-@contextmanager
-def set_environ(**kwargs: str) -> Generator[None, Any, None]:
- """Temporarily set environment variables inside the context manager and
- fully restore previous environment afterwards
- """
- ...
-
-def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
- """Similar to ``ast.walk``, but walks only function body and skips nested
- functions defined within the node.
- """
- ...
-
-_generator_callbacks_cache = ...
-def is_generator_with_return_value(callable: Callable) -> bool:
- """
- Returns True if a callable is a generator function which includes a
- 'return' statement with a value different than None, False otherwise
- """
- ...
-
-def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
- """
- Logs a warning if a callable is a generator function and includes
- a 'return' statement with a value different than None
- """
- ...
-
diff --git a/typings/scrapy/utils/ossignal.pyi b/typings/scrapy/utils/ossignal.pyi
deleted file mode 100644
index ca6e3ded..00000000
--- a/typings/scrapy/utils/ossignal.pyi
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import signal
-from types import FrameType
-from typing import Any, Callable, Dict, Optional, Union
-
-SignalHandlerT = Union[Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None]
-signal_names: Dict[int, str] = ...
-def install_shutdown_handlers(function: SignalHandlerT, override_sigint: bool = ...) -> None:
- """Install the given function as a signal handler for all common shutdown
- signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
- SIGINT handler won't be install if there is already a handler in place
- (e.g. Pdb)
- """
- ...
-
diff --git a/typings/scrapy/utils/project.pyi b/typings/scrapy/utils/project.pyi
deleted file mode 100644
index 6e9b1e07..00000000
--- a/typings/scrapy/utils/project.pyi
+++ /dev/null
@@ -1,25 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from scrapy.settings import Settings
-
-ENVVAR = ...
-DATADIR_CFG_SECTION = ...
-def inside_project() -> bool:
- ...
-
-def project_data_dir(project: str = ...) -> str:
- """Return the current project data dir, creating it if it doesn't exist"""
- ...
-
-def data_path(path: str, createdir: bool = ...) -> str:
- """
- Return the given path joined with the .scrapy data directory.
- If given an absolute path, return it unmodified.
- """
- ...
-
-def get_project_settings() -> Settings:
- ...
-
diff --git a/typings/scrapy/utils/python.pyi b/typings/scrapy/utils/python.pyi
deleted file mode 100644
index c6a7105f..00000000
--- a/typings/scrapy/utils/python.pyi
+++ /dev/null
@@ -1,201 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import sys
-from typing import Any, AsyncIterable, AsyncIterator, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, Pattern, Tuple, Union, overload
-
-"""
-This module contains essential stuff that should've come with Python itself ;)
-"""
-def flatten(x: Iterable) -> list:
- """flatten(sequence) -> list
-
- Returns a single, flat list which contains all elements retrieved
- from the sequence and all recursively contained sub-sequences
- (iterables).
-
- Examples:
- >>> [1, 2, [3,4], (5,6)]
- [1, 2, [3, 4], (5, 6)]
- >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
- [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
- >>> flatten(["foo", "bar"])
- ['foo', 'bar']
- >>> flatten(["foo", ["baz", 42], "bar"])
- ['foo', 'baz', 42, 'bar']
- """
- ...
-
-def iflatten(x: Iterable) -> Iterable:
- """iflatten(sequence) -> iterator
-
- Similar to ``.flatten()``, but returns iterator instead"""
- ...
-
-def is_listlike(x: Any) -> bool:
- """
- >>> is_listlike("foo")
- False
- >>> is_listlike(5)
- False
- >>> is_listlike(b"foo")
- False
- >>> is_listlike([b"foo"])
- True
- >>> is_listlike((b"foo",))
- True
- >>> is_listlike({})
- True
- >>> is_listlike(set())
- True
- >>> is_listlike((x for x in range(3)))
- True
- >>> is_listlike(range(5))
- True
- """
- ...
-
-def unique(list_: Iterable, key: Callable[[Any], Any] = ...) -> list:
- """efficient function to uniquify a list preserving item order"""
- ...
-
-def to_unicode(text: Union[str, bytes], encoding: Optional[str] = ..., errors: str = ...) -> str:
- """Return the unicode representation of a bytes object ``text``. If
- ``text`` is already an unicode object, return it as-is."""
- ...
-
-def to_bytes(text: Union[str, bytes], encoding: Optional[str] = ..., errors: str = ...) -> bytes:
- """Return the binary representation of ``text``. If ``text``
- is already a bytes object, return it as-is."""
- ...
-
-def re_rsearch(pattern: Union[str, Pattern], text: str, chunk_size: int = ...) -> Optional[Tuple[int, int]]:
- """
- This function does a reverse search in a text using a regular expression
- given in the attribute 'pattern'.
- Since the re module does not provide this functionality, we have to find for
- the expression into chunks of text extracted from the end (for the sake of efficiency).
- At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for
- the pattern. If the pattern is not found, another chunk is extracted, and another
- search is performed.
- This process continues until a match is found, or until the whole file is read.
- In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
- the start position of the match, and the ending (regarding the entire text).
- """
- ...
-
-def memoizemethod_noargs(method: Callable) -> Callable:
- """Decorator to cache the result of a method (without arguments) using a
- weak reference to its object
- """
- ...
-
-_BINARYCHARS = ...
-def binary_is_text(data: bytes) -> bool:
- """Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
- does not contain unprintable control characters.
- """
- ...
-
-def get_func_args(func: Callable, stripself: bool = ...) -> List[str]:
- """Return the argument name list of a callable object"""
- ...
-
-def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
- """Returns (args, kwargs) tuple for a function
- >>> import re
- >>> get_spec(re.match)
- (['pattern', 'string'], {'flags': 0})
-
- >>> class Test:
- ... def __call__(self, val):
- ... pass
- ... def method(self, val, flags=0):
- ... pass
-
- >>> get_spec(Test)
- (['self', 'val'], {})
-
- >>> get_spec(Test.method)
- (['self', 'val'], {'flags': 0})
-
- >>> get_spec(Test().method)
- (['self', 'val'], {'flags': 0})
- """
- ...
-
-def equal_attributes(obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]) -> bool:
- """Compare two objects attributes"""
- ...
-
-@overload
-def without_none_values(iterable: Mapping) -> dict:
- ...
-
-@overload
-def without_none_values(iterable: Iterable) -> Iterable:
- ...
-
-def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
- """Return a copy of ``iterable`` with all ``None`` entries removed.
-
- If ``iterable`` is a mapping, return a dictionary where all pairs that have
- value ``None`` have been removed.
- """
- ...
-
-def global_object_name(obj: Any) -> str:
- """
- Return full name of a global object.
-
- >>> from scrapy import Request
- >>> global_object_name(Request)
- 'scrapy.http.request.Request'
- """
- ...
-
-if hasattr(sys, "pypy_version_info"):
- def garbage_collect() -> None:
- ...
-
-else:
- def garbage_collect() -> None:
- ...
-
-class MutableChain(Iterable):
- """
- Thin wrapper around itertools.chain, allowing to add iterables "in-place"
- """
- def __init__(self, *args: Iterable) -> None:
- ...
-
- def extend(self, *iterables: Iterable) -> None:
- ...
-
- def __iter__(self) -> Iterator:
- ...
-
- def __next__(self) -> Any:
- ...
-
-
-
-class MutableAsyncChain(AsyncIterable):
- """
- Similar to MutableChain but for async iterables
- """
- def __init__(self, *args: Union[Iterable, AsyncIterable]) -> None:
- ...
-
- def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
- ...
-
- def __aiter__(self) -> AsyncIterator:
- ...
-
- async def __anext__(self) -> Any:
- ...
-
-
-
diff --git a/typings/scrapy/utils/reactor.pyi b/typings/scrapy/utils/reactor.pyi
deleted file mode 100644
index 8ecd8247..00000000
--- a/typings/scrapy/utils/reactor.pyi
+++ /dev/null
@@ -1,61 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
-from typing import Any, Callable, Optional
-
-def listen_tcp(portrange, host, factory): # -> None:
- """Like reactor.listenTCP but tries different ports in a range."""
- ...
-
-class CallLaterOnce:
- """Schedule a function to be called in the next reactor loop, but only if
- it hasn't been already scheduled since the last time it ran.
- """
- def __init__(self, func: Callable, *a: Any, **kw: Any) -> None:
- ...
-
- def schedule(self, delay: float = ...) -> None:
- ...
-
- def cancel(self) -> None:
- ...
-
- def __call__(self) -> Any:
- ...
-
-
-
-def set_asyncio_event_loop_policy() -> None:
- """The policy functions from asyncio often behave unexpectedly,
- so we restrict their use to the absolutely essential case.
- This should only be used to install the reactor.
- """
- ...
-
-def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
- ...
-
-def install_reactor(reactor_path: str, event_loop_path: Optional[str] = ...) -> None:
- """Installs the :mod:`~twisted.internet.reactor` with the specified
- import path. Also installs the asyncio event loop with the specified import
- path if the asyncio reactor is enabled"""
- ...
-
-def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop:
- """Sets and returns the event loop with specified import path."""
- ...
-
-def verify_installed_reactor(reactor_path: str) -> None:
- """Raises :exc:`Exception` if the installed
- :mod:`~twisted.internet.reactor` does not match the specified import
- path."""
- ...
-
-def verify_installed_asyncio_event_loop(loop_path: str) -> None:
- ...
-
-def is_asyncio_reactor_installed() -> bool:
- ...
-
diff --git a/typings/scrapy/utils/request.pyi b/typings/scrapy/utils/request.pyi
deleted file mode 100644
index 1cdd56fb..00000000
--- a/typings/scrapy/utils/request.pyi
+++ /dev/null
@@ -1,150 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Dict, Iterable, Optional, Protocol, TYPE_CHECKING, Tuple, Union
-from weakref import WeakKeyDictionary
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
-
-"""
-This module provides some useful functions for working with
-scrapy.http.Request objects
-"""
-if TYPE_CHECKING:
- ...
-_deprecated_fingerprint_cache: WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]
-_deprecated_fingerprint_cache = ...
-def request_fingerprint(request: Request, include_headers: Optional[Iterable[Union[bytes, str]]] = ..., keep_fragments: bool = ...) -> str:
- """
- Return the request fingerprint as an hexadecimal string.
-
- The request fingerprint is a hash that uniquely identifies the resource the
- request points to. For example, take the following two urls:
-
- http://www.example.com/query?id=111&cat=222
- http://www.example.com/query?cat=222&id=111
-
- Even though those are two different URLs both point to the same resource
- and are equivalent (i.e. they should return the same response).
-
- Another example are cookies used to store session ids. Suppose the
- following page is only accessible to authenticated users:
-
- http://www.example.com/members/offers.html
-
- Lots of sites use a cookie to store the session id, which adds a random
- component to the HTTP Request and thus should be ignored when calculating
- the fingerprint.
-
- For this reason, request headers are ignored by default when calculating
- the fingerprint. If you want to include specific headers use the
- include_headers argument, which is a list of Request headers to include.
-
- Also, servers usually ignore fragments in urls when handling requests,
- so they are also ignored by default when calculating the fingerprint.
- If you want to include them, set the keep_fragments argument to True
- (for instance when handling requests with a headless browser).
- """
- ...
-
-_fingerprint_cache: WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]
-_fingerprint_cache = ...
-def fingerprint(request: Request, *, include_headers: Optional[Iterable[Union[bytes, str]]] = ..., keep_fragments: bool = ...) -> bytes:
- """
- Return the request fingerprint.
-
- The request fingerprint is a hash that uniquely identifies the resource the
- request points to. For example, take the following two urls:
-
- http://www.example.com/query?id=111&cat=222
- http://www.example.com/query?cat=222&id=111
-
- Even though those are two different URLs both point to the same resource
- and are equivalent (i.e. they should return the same response).
-
- Another example are cookies used to store session ids. Suppose the
- following page is only accessible to authenticated users:
-
- http://www.example.com/members/offers.html
-
- Lots of sites use a cookie to store the session id, which adds a random
- component to the HTTP Request and thus should be ignored when calculating
- the fingerprint.
-
- For this reason, request headers are ignored by default when calculating
- the fingerprint. If you want to include specific headers use the
- include_headers argument, which is a list of Request headers to include.
-
- Also, servers usually ignore fragments in urls when handling requests,
- so they are also ignored by default when calculating the fingerprint.
- If you want to include them, set the keep_fragments argument to True
- (for instance when handling requests with a headless browser).
- """
- ...
-
-class RequestFingerprinterProtocol(Protocol):
- def fingerprint(self, request: Request) -> bytes:
- ...
-
-
-
-class RequestFingerprinter:
- """Default fingerprinter.
-
- It takes into account a canonical version
- (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
- ` and the values of :attr:`request.method
- ` and :attr:`request.body
- `. It then generates an `SHA1
- `_ hash.
-
- .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
- """
- @classmethod
- def from_crawler(cls, crawler): # -> Self:
- ...
-
- def __init__(self, crawler: Optional[Crawler] = ...) -> None:
- ...
-
- def fingerprint(self, request: Request) -> bytes:
- ...
-
-
-
-def request_authenticate(request: Request, username: str, password: str) -> None:
- """Authenticate the given request (in place) using the HTTP basic access
- authentication mechanism (RFC 2617) and the given username and password
- """
- ...
-
-def request_httprepr(request: Request) -> bytes:
- """Return the raw HTTP representation (as bytes) of the given request.
- This is provided only for reference since it's not the actual stream of
- bytes that will be send when performing the request (that's controlled
- by Twisted).
- """
- ...
-
-def referer_str(request: Request) -> Optional[str]:
- """Return Referer HTTP header suitable for logging."""
- ...
-
-def request_from_dict(d: dict, *, spider: Optional[Spider] = ...) -> Request:
- """Create a :class:`~scrapy.Request` object from a dict.
-
- If a spider is given, it will try to resolve the callbacks looking at the
- spider for methods with the same name.
- """
- ...
-
-def request_to_curl(request: Request) -> str:
- """
- Converts a :class:`~scrapy.Request` object to a curl command.
-
- :param :class:`~scrapy.Request`: Request object to be converted
- :return: string containing the curl command
- """
- ...
-
diff --git a/typings/scrapy/utils/response.pyi b/typings/scrapy/utils/response.pyi
deleted file mode 100644
index b9de172c..00000000
--- a/typings/scrapy/utils/response.pyi
+++ /dev/null
@@ -1,44 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import scrapy
-from typing import Any, Callable, Iterable, Tuple, Union
-from weakref import WeakKeyDictionary
-from twisted.web import http
-from w3lib import html
-from scrapy.http.response import Response
-from scrapy.utils.decorators import deprecated
-
-"""
-This module provides some useful functions for working with
-scrapy.http.Response objects
-"""
-_baseurl_cache: WeakKeyDictionary[Response, str] = ...
-def get_base_url(response: scrapy.http.response.text.TextResponse) -> str:
- """Return the base url of the given response, joined with the response url"""
- ...
-
-_metaref_cache: WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]] = ...
-def get_meta_refresh(response: scrapy.http.response.text.TextResponse, ignore_tags: Iterable[str] = ...) -> Union[Tuple[None, None], Tuple[float, str]]:
- """Parse the http-equiv refresh parameter from the given response"""
- ...
-
-def response_status_message(status: Union[bytes, float, int, str]) -> str:
- """Return status code plus status text descriptive message"""
- ...
-
-@deprecated
-def response_httprepr(response: Response) -> bytes:
- """Return raw HTTP representation (as bytes) of the given response. This
- is provided only for reference, since it's not the exact stream of bytes
- that was received (that's not exposed by Twisted).
- """
- ...
-
-def open_in_browser(response: Union[scrapy.http.response.html.HtmlResponse, scrapy.http.response.text.TextResponse,], _openfunc: Callable[[str], Any] = ...) -> Any:
- """Open the given response in a local web browser, populating the
- tag for external links to work
- """
- ...
-
diff --git a/typings/scrapy/utils/serialize.pyi b/typings/scrapy/utils/serialize.pyi
deleted file mode 100644
index 7baa404e..00000000
--- a/typings/scrapy/utils/serialize.pyi
+++ /dev/null
@@ -1,19 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import json
-from typing import Any
-
-class ScrapyJSONEncoder(json.JSONEncoder):
- DATE_FORMAT = ...
- TIME_FORMAT = ...
- def default(self, o: Any) -> Any:
- ...
-
-
-
-class ScrapyJSONDecoder(json.JSONDecoder):
- ...
-
-
diff --git a/typings/scrapy/utils/signal.pyi b/typings/scrapy/utils/signal.pyi
deleted file mode 100644
index 6a92c416..00000000
--- a/typings/scrapy/utils/signal.pyi
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any as TypingAny, List, Tuple
-from twisted.internet.defer import Deferred
-
-"""Helper functions for working with signals"""
-logger = ...
-def send_catch_log(signal: TypingAny = ..., sender: TypingAny = ..., *arguments: TypingAny, **named: TypingAny) -> List[Tuple[TypingAny, TypingAny]]:
- """Like pydispatcher.robust.sendRobust but it also logs errors and returns
- Failures instead of exceptions.
- """
- ...
-
-def send_catch_log_deferred(signal: TypingAny = ..., sender: TypingAny = ..., *arguments: TypingAny, **named: TypingAny) -> Deferred:
- """Like send_catch_log but supports returning deferreds on signal handlers.
- Returns a deferred that gets fired once all signal handlers deferreds were
- fired.
- """
- ...
-
-def disconnect_all(signal: TypingAny = ..., sender: TypingAny = ...) -> None:
- """Disconnect all signal handlers. Useful for cleaning up after running
- tests
- """
- ...
-
diff --git a/typings/scrapy/utils/sitemap.pyi b/typings/scrapy/utils/sitemap.pyi
deleted file mode 100644
index 23c1d6a7..00000000
--- a/typings/scrapy/utils/sitemap.pyi
+++ /dev/null
@@ -1,29 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Dict, Generator, Iterator, Optional
-
-"""
-Module for processing Sitemaps.
-
-Note: The main purpose of this module is to provide support for the
-SitemapSpider, its API is subject to change without notice.
-"""
-class Sitemap:
- """Class to parse Sitemap (type=urlset) and Sitemap Index
- (type=sitemapindex) files"""
- def __init__(self, xmltext: str) -> None:
- ...
-
- def __iter__(self) -> Iterator[Dict[str, Any]]:
- ...
-
-
-
-def sitemap_urls_from_robots(robots_text: str, base_url: Optional[str] = ...) -> Generator[str, Any, None]:
- """Return an iterator over all sitemap urls contained in the given
- robots.txt file
- """
- ...
-
diff --git a/typings/scrapy/utils/spider.pyi b/typings/scrapy/utils/spider.pyi
deleted file mode 100644
index f8fb56c0..00000000
--- a/typings/scrapy/utils/spider.pyi
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from types import CoroutineType, ModuleType
-from typing import Any, AsyncGenerator, Generator, Iterable, Literal, Optional, TYPE_CHECKING, Type, TypeVar, Union, overload
-from twisted.internet.defer import Deferred
-from scrapy import Request
-from scrapy.spiders import Spider
-from scrapy.spiderloader import SpiderLoader
-
-if TYPE_CHECKING:
- ...
-logger = ...
-_T = TypeVar("_T")
-@overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:
- ...
-
-@overload
-def iterate_spider_output(result: CoroutineType) -> Deferred:
- ...
-
-@overload
-def iterate_spider_output(result: _T) -> Iterable:
- ...
-
-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
- ...
-
-def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
- """Return an iterator over all spider classes defined in the given module
- that can be instantiated (i.e. which have name)
- """
- ...
-
-@overload
-def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ...) -> Type[Spider]:
- ...
-
-@overload
-def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]:
- ...
-
-@overload
-def spidercls_for_request(spider_loader: SpiderLoader, request: Request, *, log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]:
- ...
-
-def spidercls_for_request(spider_loader: SpiderLoader, request: Request, default_spidercls: Optional[Type[Spider]] = ..., log_none: bool = ..., log_multiple: bool = ...) -> Optional[Type[Spider]]:
- """Return a spider class that handles the given Request.
-
- This will look for the spiders that can handle the given request (using
- the spider loader) and return a Spider class if (and only if) there is
- only one Spider able to handle the Request.
-
- If multiple spiders (or no spider) are found, it will return the
- default_spidercls passed. It can optionally log if multiple or no spiders
- are found.
- """
- ...
-
-class DefaultSpider(Spider):
- name = ...
-
-
diff --git a/typings/scrapy/utils/ssl.pyi b/typings/scrapy/utils/ssl.pyi
deleted file mode 100644
index c66dc976..00000000
--- a/typings/scrapy/utils/ssl.pyi
+++ /dev/null
@@ -1,19 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, Optional
-from OpenSSL.crypto import X509Name
-
-def ffi_buf_to_string(buf: Any) -> str:
- ...
-
-def x509name_to_string(x509name: X509Name) -> str:
- ...
-
-def get_temp_key_info(ssl_object: Any) -> Optional[str]:
- ...
-
-def get_openssl_version() -> str:
- ...
-
diff --git a/typings/scrapy/utils/trackref.pyi b/typings/scrapy/utils/trackref.pyi
deleted file mode 100644
index 6cde7725..00000000
--- a/typings/scrapy/utils/trackref.pyi
+++ /dev/null
@@ -1,46 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Any, DefaultDict, Iterable, TYPE_CHECKING
-from weakref import WeakKeyDictionary
-from typing_extensions import Self
-
-"""This module provides some functions and classes to record and report
-references to live object instances.
-
-If you want live objects for a particular class to be tracked, you only have to
-subclass from object_ref (instead of object).
-
-About performance: This library has a minimal performance impact when enabled,
-and no performance penalty at all when disabled (as object_ref becomes just an
-alias to object in that case).
-"""
-if TYPE_CHECKING:
- ...
-NoneType = ...
-live_refs: DefaultDict[type, WeakKeyDictionary] = ...
-class object_ref:
- """Inherit from this class to a keep a record of live instances"""
- __slots__ = ...
- def __new__(cls, *args: Any, **kwargs: Any) -> Self:
- ...
-
-
-
-def format_live_refs(ignore: Any = ...) -> str:
- """Return a tabular representation of tracked objects"""
- ...
-
-def print_live_refs(*a: Any, **kw: Any) -> None:
- """Print tracked objects"""
- ...
-
-def get_oldest(class_name: str) -> Any:
- """Get the oldest object for a specific class name"""
- ...
-
-def iter_all(class_name: str) -> Iterable[Any]:
- """Iterate over all objects of the same class by its class name"""
- ...
-
diff --git a/typings/scrapy/utils/url.pyi b/typings/scrapy/utils/url.pyi
deleted file mode 100644
index 27862434..00000000
--- a/typings/scrapy/utils/url.pyi
+++ /dev/null
@@ -1,83 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import Iterable, Optional, TYPE_CHECKING, Type, Union
-from urllib.parse import ParseResult
-from w3lib.url import *
-from scrapy import Spider
-
-"""
-This module contains general purpose URL functions not found in the standard
-library.
-
-Some of the functions that used to be imported from this module have been moved
-to the w3lib.url module. Always import those from there instead.
-"""
-if TYPE_CHECKING:
- ...
-UrlT = Union[str, bytes, ParseResult]
-def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
- """Return True if the url belongs to any of the given domains"""
- ...
-
-def url_is_from_spider(url: UrlT, spider: Type[Spider]) -> bool:
- """Return True if the url belongs to the given spider"""
- ...
-
-def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
- """Return True if the url ends with one of the extensions provided"""
- ...
-
-def parse_url(url: UrlT, encoding: Optional[str] = ...) -> ParseResult:
- """Return urlparsed url from the given argument (which could be an already
- parsed url)
- """
- ...
-
-def escape_ajax(url: str) -> str:
- """
- Return the crawlable url according to:
- https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
-
- >>> escape_ajax("www.example.com/ajax.html#!key=value")
- 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
- >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value")
- 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue'
- >>> escape_ajax("www.example.com/ajax.html?#!key=value")
- 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
- >>> escape_ajax("www.example.com/ajax.html#!")
- 'www.example.com/ajax.html?_escaped_fragment_='
-
- URLs that are not "AJAX crawlable" (according to Google) returned as-is:
-
- >>> escape_ajax("www.example.com/ajax.html#key=value")
- 'www.example.com/ajax.html#key=value'
- >>> escape_ajax("www.example.com/ajax.html#")
- 'www.example.com/ajax.html#'
- >>> escape_ajax("www.example.com/ajax.html")
- 'www.example.com/ajax.html'
- """
- ...
-
-def add_http_if_no_scheme(url: str) -> str:
- """Add http as the default scheme if it is missing from the url."""
- ...
-
-def guess_scheme(url: str) -> str:
- """Add an URL scheme if missing: file:// for filepath-like input or
- http:// otherwise."""
- ...
-
-def strip_url(url: str, strip_credentials: bool = ..., strip_default_port: bool = ..., origin_only: bool = ..., strip_fragment: bool = ...) -> str:
- """Strip URL string from some of its components:
-
- - ``strip_credentials`` removes "user:password@"
- - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
- from http:// (resp. https://, ftp://) URLs
- - ``origin_only`` replaces path component with "/", also dropping
- query and fragment components ; it also strips credentials
- - ``strip_fragment`` drops any #fragment component
- """
- ...
-
diff --git a/typings/scrapy/utils/versions.pyi b/typings/scrapy/utils/versions.pyi
deleted file mode 100644
index cf99dbbb..00000000
--- a/typings/scrapy/utils/versions.pyi
+++ /dev/null
@@ -1,9 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from typing import List, Tuple
-
-def scrapy_components_versions() -> List[Tuple[str, str]]:
- ...
-
diff --git a/typings/toolz/__init__.pyi b/typings/toolz/__init__.pyi
deleted file mode 100644
index 1108f52b..00000000
--- a/typings/toolz/__init__.pyi
+++ /dev/null
@@ -1,17 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from .itertoolz import *
-from .functoolz import *
-from .dicttoolz import *
-from .recipes import *
-from functools import partial, reduce
-from . import curried, sandbox
-from ._version import get_versions
-
-sorted = ...
-map = map
-filter = filter
-comp = ...
-__version__ = get_versions()['version']
diff --git a/typings/toolz/_signatures.pyi b/typings/toolz/_signatures.pyi
deleted file mode 100644
index a01daa22..00000000
--- a/typings/toolz/_signatures.pyi
+++ /dev/null
@@ -1,73 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""Internal module for better introspection of builtins.
-
-The main functions are ``is_builtin_valid_args``, ``is_builtin_partial_args``,
-and ``has_unknown_args``. Other functions in this module support these three.
-
-Notably, we create a ``signatures`` registry to enable introspection of
-builtin functions in any Python version. This includes builtins that
-have more than one valid signature. Currently, the registry includes
-builtins from ``builtins``, ``functools``, ``itertools``, and ``operator``
-modules. More can be added as requested. We don't guarantee full coverage.
-
-Everything in this module should be regarded as implementation details.
-Users should try to not use this module directly.
-"""
-module_info = ...
-def num_pos_args(sigspec): # -> int:
- """ Return the number of positional arguments. ``f(x, y=1)`` has 1"""
- ...
-
-def get_exclude_keywords(num_pos_only, sigspec): # -> tuple[()] | tuple[Unknown, ...]:
- """ Return the names of position-only arguments if func has **kwargs"""
- ...
-
-def signature_or_spec(func): # -> Signature | None:
- ...
-
-def expand_sig(sig): # -> tuple[Unknown | int, Unknown, Unknown | tuple[Any, ...], Signature | None]:
- """ Convert the signature spec in ``module_info`` to add to ``signatures``
-
- The input signature spec is one of:
- - ``lambda_func``
- - ``(num_position_args, lambda_func)``
- - ``(num_position_args, lambda_func, keyword_only_args)``
-
- The output signature spec is:
- ``(num_position_args, lambda_func, keyword_exclude, sigspec)``
-
- where ``keyword_exclude`` includes keyword only arguments and, if variadic
- keywords is present, the names of position-only argument. The latter is
- included to support builtins such as ``partial(func, *args, **kwargs)``,
- which allows ``func=`` to be used as a keyword even though it's the name
- of a positional argument.
- """
- ...
-
-signatures = ...
-def create_signature_registry(module_info=..., signatures=...): # -> None:
- ...
-
-def check_valid(sig, args, kwargs): # -> bool:
- """ Like ``is_valid_args`` for the given signature spec"""
- ...
-
-def check_partial(sig, args, kwargs):
- """ Like ``is_partial_args`` for the given signature spec"""
- ...
-
-def check_arity(n, sig): # -> bool | None:
- ...
-
-def check_varargs(sig): # -> bool | None:
- ...
-
-def check_keywords(sig): # -> bool | None:
- ...
-
-def check_required_args(sig): # -> int | Literal[False] | None:
- ...
-
diff --git a/typings/toolz/_version.pyi b/typings/toolz/_version.pyi
deleted file mode 100644
index 28e6f13f..00000000
--- a/typings/toolz/_version.pyi
+++ /dev/null
@@ -1,8 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-version_json = ...
-def get_versions(): # -> Any:
- ...
-
diff --git a/typings/toolz/curried/__init__.pyi b/typings/toolz/curried/__init__.pyi
deleted file mode 100644
index 321c6226..00000000
--- a/typings/toolz/curried/__init__.pyi
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import toolz
-from . import operator
-from toolz import apply, comp, complement, compose, compose_left, concat, concatv, count, curry, diff, first, flip, frequencies, identity, interleave, isdistinct, isiterable, juxt, last, memoize, merge_sorted, peek, pipe, second, thread_first, thread_last
-from .exceptions import merge, merge_with
-
-"""
-Alternate namespace for toolz such that all functions are curried
-
-Currying provides implicit partial evaluation of all functions
-
-Example:
-
- Get usually requires two arguments, an index and a collection
- >>> from toolz.curried import get
- >>> get(0, ('a', 'b'))
- 'a'
-
- When we use it in higher order functions we often want to pass a partially
- evaluated form
- >>> data = [(1, 2), (11, 22), (111, 222)]
- >>> list(map(lambda seq: get(0, seq), data))
- [1, 11, 111]
-
- The curried version allows simple expression of partial evaluation
- >>> list(map(get(0), data))
- [1, 11, 111]
-
-See Also:
- toolz.functoolz.curry
-"""
-accumulate = ...
-assoc = ...
-assoc_in = ...
-cons = ...
-countby = ...
-dissoc = ...
-do = ...
-drop = ...
-excepts = ...
-filter = ...
-get = ...
-get_in = ...
-groupby = ...
-interpose = ...
-itemfilter = ...
-itemmap = ...
-iterate = ...
-join = ...
-keyfilter = ...
-keymap = ...
-map = ...
-mapcat = ...
-nth = ...
-partial = ...
-partition = ...
-partition_all = ...
-partitionby = ...
-peekn = ...
-pluck = ...
-random_sample = ...
-reduce = ...
-reduceby = ...
-remove = ...
-sliding_window = ...
-sorted = ...
-tail = ...
-take = ...
-take_nth = ...
-topk = ...
-unique = ...
-update_in = ...
-valfilter = ...
-valmap = ...
diff --git a/typings/toolz/curried/exceptions.pyi b/typings/toolz/curried/exceptions.pyi
deleted file mode 100644
index 85684966..00000000
--- a/typings/toolz/curried/exceptions.pyi
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-import toolz
-
-__all__ = ['merge_with', 'merge']
-@toolz.curry
-def merge_with(func, d, *dicts, **kwargs): # -> dict[Unknown, Unknown]:
- ...
-
-@toolz.curry
-def merge(d, *dicts, **kwargs): # -> dict[Unknown, Unknown]:
- ...
-
diff --git a/typings/toolz/curried/operator.pyi b/typings/toolz/curried/operator.pyi
deleted file mode 100644
index aed5435d..00000000
--- a/typings/toolz/curried/operator.pyi
+++ /dev/null
@@ -1,5 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-IGNORE = ...
diff --git a/typings/toolz/dicttoolz.pyi b/typings/toolz/dicttoolz.pyi
deleted file mode 100644
index f16f92ca..00000000
--- a/typings/toolz/dicttoolz.pyi
+++ /dev/null
@@ -1,235 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-__all__ = ('merge', 'merge_with', 'valmap', 'keymap', 'itemmap', 'valfilter', 'keyfilter', 'itemfilter', 'assoc', 'dissoc', 'assoc_in', 'update_in', 'get_in')
-def merge(*dicts, **kwargs): # -> dict[Unknown, Unknown]:
- """ Merge a collection of dictionaries
-
- >>> merge({1: 'one'}, {2: 'two'})
- {1: 'one', 2: 'two'}
-
- Later dictionaries have precedence
-
- >>> merge({1: 2, 3: 4}, {3: 3, 4: 4})
- {1: 2, 3: 3, 4: 4}
-
- See Also:
- merge_with
- """
- ...
-
-def merge_with(func, *dicts, **kwargs): # -> dict[Unknown, Unknown]:
- """ Merge dictionaries and apply function to combined values
-
- A key may occur in more than one dict, and all values mapped from the key
- will be passed to the function as a list, such as func([val1, val2, ...]).
-
- >>> merge_with(sum, {1: 1, 2: 2}, {1: 10, 2: 20})
- {1: 11, 2: 22}
-
- >>> merge_with(first, {1: 1, 2: 2}, {2: 20, 3: 30}) # doctest: +SKIP
- {1: 1, 2: 2, 3: 30}
-
- See Also:
- merge
- """
- ...
-
-def valmap(func, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Apply function to values of dictionary
-
- >>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]}
- >>> valmap(sum, bills) # doctest: +SKIP
- {'Alice': 65, 'Bob': 45}
-
- See Also:
- keymap
- itemmap
- """
- ...
-
-def keymap(func, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Apply function to keys of dictionary
-
- >>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]}
- >>> keymap(str.lower, bills) # doctest: +SKIP
- {'alice': [20, 15, 30], 'bob': [10, 35]}
-
- See Also:
- valmap
- itemmap
- """
- ...
-
-def itemmap(func, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Apply function to items of dictionary
-
- >>> accountids = {"Alice": 10, "Bob": 20}
- >>> itemmap(reversed, accountids) # doctest: +SKIP
- {10: "Alice", 20: "Bob"}
-
- See Also:
- keymap
- valmap
- """
- ...
-
-def valfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Filter items in dictionary by value
-
- >>> iseven = lambda x: x % 2 == 0
- >>> d = {1: 2, 2: 3, 3: 4, 4: 5}
- >>> valfilter(iseven, d)
- {1: 2, 3: 4}
-
- See Also:
- keyfilter
- itemfilter
- valmap
- """
- ...
-
-def keyfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Filter items in dictionary by key
-
- >>> iseven = lambda x: x % 2 == 0
- >>> d = {1: 2, 2: 3, 3: 4, 4: 5}
- >>> keyfilter(iseven, d)
- {2: 3, 4: 5}
-
- See Also:
- valfilter
- itemfilter
- keymap
- """
- ...
-
-def itemfilter(predicate, d, factory=...): # -> dict[Unknown, Unknown]:
- """ Filter items in dictionary by item
-
- >>> def isvalid(item):
- ... k, v = item
- ... return k % 2 == 0 and v < 4
-
- >>> d = {1: 2, 2: 3, 3: 4, 4: 5}
- >>> itemfilter(isvalid, d)
- {2: 3}
-
- See Also:
- keyfilter
- valfilter
- itemmap
- """
- ...
-
-def assoc(d, key, value, factory=...): # -> dict[Unknown, Unknown]:
- """ Return a new dict with new key value pair
-
- New dict has d[key] set to value. Does not modify the initial dictionary.
-
- >>> assoc({'x': 1}, 'x', 2)
- {'x': 2}
- >>> assoc({'x': 1}, 'y', 3) # doctest: +SKIP
- {'x': 1, 'y': 3}
- """
- ...
-
-def dissoc(d, *keys, **kwargs): # -> dict[Unknown, Unknown]:
- """ Return a new dict with the given key(s) removed.
-
- New dict has d[key] deleted for each supplied key.
- Does not modify the initial dictionary.
-
- >>> dissoc({'x': 1, 'y': 2}, 'y')
- {'x': 1}
- >>> dissoc({'x': 1, 'y': 2}, 'y', 'x')
- {}
- >>> dissoc({'x': 1}, 'y') # Ignores missing keys
- {'x': 1}
- """
- ...
-
-def assoc_in(d, keys, value, factory=...): # -> dict[Unknown, Unknown]:
- """ Return a new dict with new, potentially nested, key value pair
-
- >>> purchase = {'name': 'Alice',
- ... 'order': {'items': ['Apple', 'Orange'],
- ... 'costs': [0.50, 1.25]},
- ... 'credit card': '5555-1234-1234-1234'}
- >>> assoc_in(purchase, ['order', 'costs'], [0.25, 1.00]) # doctest: +SKIP
- {'credit card': '5555-1234-1234-1234',
- 'name': 'Alice',
- 'order': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}}
- """
- ...
-
-def update_in(d, keys, func, default=..., factory=...): # -> dict[Unknown, Unknown]:
- """ Update value in a (potentially) nested dictionary
-
- inputs:
- d - dictionary on which to operate
- keys - list or tuple giving the location of the value to be changed in d
- func - function to operate on that value
-
- If keys == [k0,..,kX] and d[k0]..[kX] == v, update_in returns a copy of the
- original dictionary with v replaced by func(v), but does not mutate the
- original dictionary.
-
- If k0 is not a key in d, update_in creates nested dictionaries to the depth
- specified by the keys, with the innermost value set to func(default).
-
- >>> inc = lambda x: x + 1
- >>> update_in({'a': 0}, ['a'], inc)
- {'a': 1}
-
- >>> transaction = {'name': 'Alice',
- ... 'purchase': {'items': ['Apple', 'Orange'],
- ... 'costs': [0.50, 1.25]},
- ... 'credit card': '5555-1234-1234-1234'}
- >>> update_in(transaction, ['purchase', 'costs'], sum) # doctest: +SKIP
- {'credit card': '5555-1234-1234-1234',
- 'name': 'Alice',
- 'purchase': {'costs': 1.75, 'items': ['Apple', 'Orange']}}
-
- >>> # updating a value when k0 is not in d
- >>> update_in({}, [1, 2, 3], str, default="bar")
- {1: {2: {3: 'bar'}}}
- >>> update_in({1: 'foo'}, [2, 3, 4], inc, 0)
- {1: 'foo', 2: {3: {4: 1}}}
- """
- ...
-
-def get_in(keys, coll, default=..., no_default=...): # -> Sequence[Unknown] | None:
- """ Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys.
-
- If coll[i0][i1]...[iX] cannot be found, returns ``default``, unless
- ``no_default`` is specified, then it raises KeyError or IndexError.
-
- ``get_in`` is a generalization of ``operator.getitem`` for nested data
- structures such as dictionaries and lists.
-
- >>> transaction = {'name': 'Alice',
- ... 'purchase': {'items': ['Apple', 'Orange'],
- ... 'costs': [0.50, 1.25]},
- ... 'credit card': '5555-1234-1234-1234'}
- >>> get_in(['purchase', 'items', 0], transaction)
- 'Apple'
- >>> get_in(['name'], transaction)
- 'Alice'
- >>> get_in(['purchase', 'total'], transaction)
- >>> get_in(['purchase', 'items', 'apple'], transaction)
- >>> get_in(['purchase', 'items', 10], transaction)
- >>> get_in(['purchase', 'total'], transaction, 0)
- 0
- >>> get_in(['y'], {}, no_default=True)
- Traceback (most recent call last):
- ...
- KeyError: 'y'
-
- See Also:
- itertoolz.get
- operator.getitem
- """
- ...
-
diff --git a/typings/toolz/functoolz.pyi b/typings/toolz/functoolz.pyi
deleted file mode 100644
index 89bdf91d..00000000
--- a/typings/toolz/functoolz.pyi
+++ /dev/null
@@ -1,540 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-"""
-This type stub file was generated by pyright.
-"""
-PYPY = ...
-__all__ = ('identity', 'apply', 'thread_first', 'thread_last', 'memoize', 'compose', 'compose_left', 'pipe', 'complement', 'juxt', 'do', 'curry', 'flip', 'excepts')
-PYPY = ...
-def identity(x):
- """ Identity function. Return x
-
- >>> identity(3)
- 3
- """
- ...
-
-def apply(*func_and_args, **kwargs):
- """ Applies a function and returns the results
-
- >>> def double(x): return 2*x
- >>> def inc(x): return x + 1
- >>> apply(double, 5)
- 10
-
- >>> tuple(map(apply, [double, inc, double], [10, 500, 8000]))
- (20, 501, 16000)
- """
- ...
-
-def thread_first(val, *forms):
- """ Thread value through a sequence of functions/forms
-
- >>> def double(x): return 2*x
- >>> def inc(x): return x + 1
- >>> thread_first(1, inc, double)
- 4
-
- If the function expects more than one input you can specify those inputs
- in a tuple. The value is used as the first input.
-
- >>> def add(x, y): return x + y
- >>> def pow(x, y): return x**y
- >>> thread_first(1, (add, 4), (pow, 2)) # pow(add(1, 4), 2)
- 25
-
- So in general
- thread_first(x, f, (g, y, z))
- expands to
- g(f(x), y, z)
-
- See Also:
- thread_last
- """
- ...
-
-def thread_last(val, *forms):
- """ Thread value through a sequence of functions/forms
-
- >>> def double(x): return 2*x
- >>> def inc(x): return x + 1
- >>> thread_last(1, inc, double)
- 4
-
- If the function expects more than one input you can specify those inputs
- in a tuple. The value is used as the last input.
-
- >>> def add(x, y): return x + y
- >>> def pow(x, y): return x**y
- >>> thread_last(1, (add, 4), (pow, 2)) # pow(2, add(4, 1))
- 32
-
- So in general
- thread_last(x, f, (g, y, z))
- expands to
- g(y, z, f(x))
-
- >>> def iseven(x):
- ... return x % 2 == 0
- >>> list(thread_last([1, 2, 3], (map, inc), (filter, iseven)))
- [2, 4]
-
- See Also:
- thread_first
- """
- ...
-
-def instanceproperty(fget=..., fset=..., fdel=..., doc=..., classval=...):
- """ Like @property, but returns ``classval`` when used as a class attribute
-
- >>> class MyClass(object):
- ... '''The class docstring'''
- ... @instanceproperty(classval=__doc__)
- ... def __doc__(self):
- ... return 'An object docstring'
- ... @instanceproperty
- ... def val(self):
- ... return 42
- ...
- >>> MyClass.__doc__
- 'The class docstring'
- >>> MyClass.val is None
- True
- >>> obj = MyClass()
- >>> obj.__doc__
- 'An object docstring'
- >>> obj.val
- 42
- """
- ...
-
-class InstanceProperty(property):
- """ Like @property, but returns ``classval`` when used as a class attribute
-
- Should not be used directly. Use ``instanceproperty`` instead.
- """
- def __init__(self, fget=..., fset=..., fdel=..., doc=..., classval=...) -> None:
- ...
-
- def __get__(self, obj, type=...):
- ...
-
- def __reduce__(self):
- ...
-
-
-
-class curry:
- """ Curry a callable function
-
- Enables partial application of arguments through calling a function with an
- incomplete set of arguments.
-
- >>> def mul(x, y):
- ... return x * y
- >>> mul = curry(mul)
-
- >>> double = mul(2)
- >>> double(10)
- 20
-
- Also supports keyword arguments
-
- >>> @curry # Can use curry as a decorator
- ... def f(x, y, a=10):
- ... return a * (x + y)
-
- >>> add = f(a=1)
- >>> add(2, 3)
- 5
-
- See Also:
- toolz.curried - namespace of curried functions
- https://toolz.readthedocs.io/en/latest/curry.html
- """
- def __init__(self, *args, **kwargs) -> None:
- ...
-
- @instanceproperty
- def func(self):
- ...
-
- @instanceproperty
- def __signature__(self):
- ...
-
- @instanceproperty
- def args(self):
- ...
-
- @instanceproperty
- def keywords(self):
- ...
-
- @instanceproperty
- def func_name(self):
- ...
-
- def __str__(self) -> str:
- ...
-
- def __repr__(self):
- ...
-
- def __hash__(self) -> int:
- ...
-
- def __eq__(self, other) -> bool:
- ...
-
- def __ne__(self, other) -> bool:
- ...
-
- def __call__(self, *args, **kwargs):
- ...
-
- def bind(self, *args, **kwargs):
- ...
-
- def call(self, *args, **kwargs):
- ...
-
- def __get__(self, instance, owner):
- ...
-
- def __reduce__(self):
- ...
-
-
-
-@curry
-def memoize(func, cache=..., key=...):
- """ Cache a function's result for speedy future evaluation
-
- Considerations:
- Trades memory for speed.
- Only use on pure functions.
-
- >>> def add(x, y): return x + y
- >>> add = memoize(add)
-
- Or use as a decorator
-
- >>> @memoize
- ... def add(x, y):
- ... return x + y
-
- Use the ``cache`` keyword to provide a dict-like object as an initial cache
-
- >>> @memoize(cache={(1, 2): 3})
- ... def add(x, y):
- ... return x + y
-
- Note that the above works as a decorator because ``memoize`` is curried.
-
- It is also possible to provide a ``key(args, kwargs)`` function that
- calculates keys used for the cache, which receives an ``args`` tuple and
- ``kwargs`` dict as input, and must return a hashable value. However,
- the default key function should be sufficient most of the time.
-
- >>> # Use key function that ignores extraneous keyword arguments
- >>> @memoize(key=lambda args, kwargs: args)
- ... def add(x, y, verbose=False):
- ... if verbose:
- ... print('Calculating %s + %s' % (x, y))
- ... return x + y
- """
- ...
-
-class Compose:
- """ A composition of functions
-
- See Also:
- compose
- """
- __slots__ = ...
- def __init__(self, funcs) -> None:
- ...
-
- def __call__(self, *args, **kwargs):
- ...
-
- def __getstate__(self):
- ...
-
- def __setstate__(self, state):
- ...
-
- @instanceproperty(classval=__doc__)
- def __doc__(self):
- ...
-
- @property
- def __name__(self):
- ...
-
- def __repr__(self):
- ...
-
- def __eq__(self, other) -> bool:
- ...
-
- def __ne__(self, other) -> bool:
- ...
-
- def __hash__(self) -> int:
- ...
-
- def __get__(self, obj, objtype=...):
- ...
-
- @instanceproperty
- def __signature__(self):
- ...
-
- __wrapped__ = ...
-
-
-def compose(*funcs):
- """ Compose functions to operate in series.
-
- Returns a function that applies other functions in sequence.
-
- Functions are applied from right to left so that
- ``compose(f, g, h)(x, y)`` is the same as ``f(g(h(x, y)))``.
-
- If no arguments are provided, the identity function (f(x) = x) is returned.
-
- >>> inc = lambda i: i + 1
- >>> compose(str, inc)(3)
- '4'
-
- See Also:
- compose_left
- pipe
- """
- ...
-
-def compose_left(*funcs):
- """ Compose functions to operate in series.
-
- Returns a function that applies other functions in sequence.
-
- Functions are applied from left to right so that
- ``compose_left(f, g, h)(x, y)`` is the same as ``h(g(f(x, y)))``.
-
- If no arguments are provided, the identity function (f(x) = x) is returned.
-
- >>> inc = lambda i: i + 1
- >>> compose_left(inc, str)(3)
- '4'
-
- See Also:
- compose
- pipe
- """
- ...
-
-def pipe(data, *funcs):
- """ Pipe a value through a sequence of functions
-
- I.e. ``pipe(data, f, g, h)`` is equivalent to ``h(g(f(data)))``
-
- We think of the value as progressing through a pipe of several
- transformations, much like pipes in UNIX
-
- ``$ cat data | f | g | h``
-
- >>> double = lambda i: 2 * i
- >>> pipe(3, double, str)
- '6'
-
- See Also:
- compose
- compose_left
- thread_first
- thread_last
- """
- ...
-
-def complement(func):
- """ Convert a predicate function to its logical complement.
-
- In other words, return a function that, for inputs that normally
- yield True, yields False, and vice-versa.
-
- >>> def iseven(n): return n % 2 == 0
- >>> isodd = complement(iseven)
- >>> iseven(2)
- True
- >>> isodd(2)
- False
- """
- ...
-
-class juxt:
- """ Creates a function that calls several functions with the same arguments
-
- Takes several functions and returns a function that applies its arguments
- to each of those functions then returns a tuple of the results.
-
- Name comes from juxtaposition: the fact of two things being seen or placed
- close together with contrasting effect.
-
- >>> inc = lambda x: x + 1
- >>> double = lambda x: x * 2
- >>> juxt(inc, double)(10)
- (11, 20)
- >>> juxt([inc, double])(10)
- (11, 20)
- """
- __slots__ = ...
- def __init__(self, *funcs) -> None:
- ...
-
- def __call__(self, *args, **kwargs):
- ...
-
- def __getstate__(self):
- ...
-
- def __setstate__(self, state):
- ...
-
-
-
-def do(func, x):
- """ Runs ``func`` on ``x``, returns ``x``
-
- Because the results of ``func`` are not returned, only the side
- effects of ``func`` are relevant.
-
- Logging functions can be made by composing ``do`` with a storage function
- like ``list.append`` or ``file.write``
-
- >>> from toolz import compose
- >>> from toolz.curried import do
-
- >>> log = []
- >>> inc = lambda x: x + 1
- >>> inc = compose(inc, do(log.append))
- >>> inc(1)
- 2
- >>> inc(11)
- 12
- >>> log
- [1, 11]
- """
- ...
-
-@curry
-def flip(func, a, b):
- """ Call the function call with the arguments flipped
-
- This function is curried.
-
- >>> def div(a, b):
- ... return a // b
- ...
- >>> flip(div, 2, 6)
- 3
- >>> div_by_two = flip(div, 2)
- >>> div_by_two(4)
- 2
-
- This is particularly useful for built in functions and functions defined
- in C extensions that accept positional only arguments. For example:
- isinstance, issubclass.
-
- >>> data = [1, 'a', 'b', 2, 1.5, object(), 3]
- >>> only_ints = list(filter(flip(isinstance, int), data))
- >>> only_ints
- [1, 2, 3]
- """
- ...
-
-def return_none(exc):
- """ Returns None.
- """
- ...
-
-class excepts:
- """A wrapper around a function to catch exceptions and
- dispatch to a handler.
-
- This is like a functional try/except block, in the same way that
- ifexprs are functional if/else blocks.
-
- Examples
- --------
- >>> excepting = excepts(
- ... ValueError,
- ... lambda a: [1, 2].index(a),
- ... lambda _: -1,
- ... )
- >>> excepting(1)
- 0
- >>> excepting(3)
- -1
-
- Multiple exceptions and default except clause.
- >>> excepting = excepts((IndexError, KeyError), lambda a: a[0])
- >>> excepting([])
- >>> excepting([1])
- 1
- >>> excepting({})
- >>> excepting({0: 1})
- 1
- """
- def __init__(self, exc, func, handler=...) -> None:
- ...
-
- def __call__(self, *args, **kwargs):
- ...
-
- @instanceproperty(classval=__doc__)
- def __doc__(self):
- ...
-
- @property
- def __name__(self):
- ...
-
-
-
-if PYPY:
- _check_sigspec_orig = ...
-def num_required_args(func, sigspec=...):
- ...
-
-def has_varargs(func, sigspec=...):
- ...
-
-def has_keywords(func, sigspec=...):
- ...
-
-def is_valid_args(func, args, kwargs, sigspec=...):
- ...
-
-def is_partial_args(func, args, kwargs, sigspec=...):
- ...
-
-def is_arity(n, func, sigspec=...):
- """ Does a function have only n positional arguments?
-
- This function relies on introspection and does not call the function.
- Returns None if validity can't be determined.
-
- >>> def f(x):
- ... return x
- >>> is_arity(1, f)
- True
- >>> def g(x, y=1):
- ... return x + y
- >>> is_arity(1, g)
- False
- """
- ...
-
diff --git a/typings/toolz/itertoolz.pyi b/typings/toolz/itertoolz.pyi
deleted file mode 100644
index a60dbea9..00000000
--- a/typings/toolz/itertoolz.pyi
+++ /dev/null
@@ -1,666 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-__all__ = ('remove', 'accumulate', 'groupby', 'merge_sorted', 'interleave', 'unique', 'isiterable', 'isdistinct', 'take', 'drop', 'take_nth', 'first', 'second', 'nth', 'last', 'get', 'concat', 'concatv', 'mapcat', 'cons', 'interpose', 'frequencies', 'reduceby', 'iterate', 'sliding_window', 'partition', 'partition_all', 'count', 'pluck', 'join', 'tail', 'diff', 'topk', 'peek', 'peekn', 'random_sample')
-def remove(predicate, seq): # -> filterfalse[Unknown]:
- """ Return those items of sequence for which predicate(item) is False
-
- >>> def iseven(x):
- ... return x % 2 == 0
- >>> list(remove(iseven, [1, 2, 3, 4]))
- [1, 3]
- """
- ...
-
-def accumulate(binop, seq, initial=...): # -> Generator[Unknown | str, Any, None]:
- """ Repeatedly apply binary function to a sequence, accumulating results
-
- >>> from operator import add, mul
- >>> list(accumulate(add, [1, 2, 3, 4, 5]))
- [1, 3, 6, 10, 15]
- >>> list(accumulate(mul, [1, 2, 3, 4, 5]))
- [1, 2, 6, 24, 120]
-
- Accumulate is similar to ``reduce`` and is good for making functions like
- cumulative sum:
-
- >>> from functools import partial, reduce
- >>> sum = partial(reduce, add)
- >>> cumsum = partial(accumulate, add)
-
- Accumulate also takes an optional argument that will be used as the first
- value. This is similar to reduce.
-
- >>> list(accumulate(add, [1, 2, 3], -1))
- [-1, 0, 2, 5]
- >>> list(accumulate(add, [], 1))
- [1]
-
- See Also:
- itertools.accumulate : In standard itertools for Python 3.2+
- """
- ...
-
-def groupby(key, seq): # -> dict[Unknown, Unknown]:
- """ Group a collection by a key function
-
- >>> names = ['Alice', 'Bob', 'Charlie', 'Dan', 'Edith', 'Frank']
- >>> groupby(len, names) # doctest: +SKIP
- {3: ['Bob', 'Dan'], 5: ['Alice', 'Edith', 'Frank'], 7: ['Charlie']}
-
- >>> iseven = lambda x: x % 2 == 0
- >>> groupby(iseven, [1, 2, 3, 4, 5, 6, 7, 8]) # doctest: +SKIP
- {False: [1, 3, 5, 7], True: [2, 4, 6, 8]}
-
- Non-callable keys imply grouping on a member.
-
- >>> groupby('gender', [{'name': 'Alice', 'gender': 'F'},
- ... {'name': 'Bob', 'gender': 'M'},
- ... {'name': 'Charlie', 'gender': 'M'}]) # doctest:+SKIP
- {'F': [{'gender': 'F', 'name': 'Alice'}],
- 'M': [{'gender': 'M', 'name': 'Bob'},
- {'gender': 'M', 'name': 'Charlie'}]}
-
- Not to be confused with ``itertools.groupby``
-
- See Also:
- countby
- """
- ...
-
-def merge_sorted(*seqs, **kwargs): # -> Iterator[Any] | Generator[Unknown, Any, None]:
- """ Merge and sort a collection of sorted collections
-
- This works lazily and only keeps one value from each iterable in memory.
-
- >>> list(merge_sorted([1, 3, 5], [2, 4, 6]))
- [1, 2, 3, 4, 5, 6]
-
- >>> ''.join(merge_sorted('abc', 'abc', 'abc'))
- 'aaabbbccc'
-
- The "key" function used to sort the input may be passed as a keyword.
-
- >>> list(merge_sorted([2, 3], [1, 3], key=lambda x: x // 3))
- [2, 1, 3, 3]
- """
- ...
-
-def interleave(seqs): # -> Generator[Unknown, Any, None]:
- """ Interleave a sequence of sequences
-
- >>> list(interleave([[1, 2], [3, 4]]))
- [1, 3, 2, 4]
-
- >>> ''.join(interleave(('ABC', 'XY')))
- 'AXBYC'
-
- Both the individual sequences and the sequence of sequences may be infinite
-
- Returns a lazy iterator
- """
- ...
-
-def unique(seq, key=...): # -> Generator[Unknown, Any, None]:
- """ Return only unique elements of a sequence
-
- >>> tuple(unique((1, 2, 3)))
- (1, 2, 3)
- >>> tuple(unique((1, 2, 1, 3)))
- (1, 2, 3)
-
- Uniqueness can be defined by key keyword
-
- >>> tuple(unique(['cat', 'mouse', 'dog', 'hen'], key=len))
- ('cat', 'mouse')
- """
- ...
-
-def isiterable(x): # -> bool:
- """ Is x iterable?
-
- >>> isiterable([1, 2, 3])
- True
- >>> isiterable('abc')
- True
- >>> isiterable(5)
- False
- """
- ...
-
-def isdistinct(seq): # -> bool:
- """ All values in sequence are distinct
-
- >>> isdistinct([1, 2, 3])
- True
- >>> isdistinct([1, 2, 1])
- False
-
- >>> isdistinct("Hello")
- False
- >>> isdistinct("World")
- True
- """
- ...
-
-def take(n, seq): # -> islice[Unknown]:
- """ The first n elements of a sequence
-
- >>> list(take(2, [10, 20, 30, 40, 50]))
- [10, 20]
-
- See Also:
- drop
- tail
- """
- ...
-
-def tail(n, seq): # -> tuple[Unknown, ...]:
- """ The last n elements of a sequence
-
- >>> tail(2, [10, 20, 30, 40, 50])
- [40, 50]
-
- See Also:
- drop
- take
- """
- ...
-
-def drop(n, seq): # -> islice[Unknown]:
- """ The sequence following the first n elements
-
- >>> list(drop(2, [10, 20, 30, 40, 50]))
- [30, 40, 50]
-
- See Also:
- take
- tail
- """
- ...
-
-def take_nth(n, seq): # -> islice[Unknown]:
- """ Every nth item in seq
-
- >>> list(take_nth(2, [10, 20, 30, 40, 50]))
- [10, 30, 50]
- """
- ...
-
-def first(seq):
- """ The first element in a sequence
-
- >>> first('ABC')
- 'A'
- """
- ...
-
-def second(seq):
- """ The second element in a sequence
-
- >>> second('ABC')
- 'B'
- """
- ...
-
-def nth(n, seq):
- """ The nth element in a sequence
-
- >>> nth(1, 'ABC')
- 'B'
- """
- ...
-
-def last(seq):
- """ The last element in a sequence
-
- >>> last('ABC')
- 'C'
- """
- ...
-
-rest = ...
-def get(ind, seq, default=...): # -> Any | tuple[Unknown] | tuple[()] | tuple[Unknown | str, ...] | str:
- """ Get element in a sequence or dict
-
- Provides standard indexing
-
- >>> get(1, 'ABC') # Same as 'ABC'[1]
- 'B'
-
- Pass a list to get multiple values
-
- >>> get([1, 2], 'ABC') # ('ABC'[1], 'ABC'[2])
- ('B', 'C')
-
- Works on any value that supports indexing/getitem
- For example here we see that it works with dictionaries
-
- >>> phonebook = {'Alice': '555-1234',
- ... 'Bob': '555-5678',
- ... 'Charlie':'555-9999'}
- >>> get('Alice', phonebook)
- '555-1234'
-
- >>> get(['Alice', 'Bob'], phonebook)
- ('555-1234', '555-5678')
-
- Provide a default for missing values
-
- >>> get(['Alice', 'Dennis'], phonebook, None)
- ('555-1234', None)
-
- See Also:
- pluck
- """
- ...
-
-def concat(seqs): # -> chain[Unknown]:
- """ Concatenate zero or more iterables, any of which may be infinite.
-
- An infinite sequence will prevent the rest of the arguments from
- being included.
-
- We use chain.from_iterable rather than ``chain(*seqs)`` so that seqs
- can be a generator.
-
- >>> list(concat([[], [1], [2, 3]]))
- [1, 2, 3]
-
- See also:
- itertools.chain.from_iterable equivalent
- """
- ...
-
-def concatv(*seqs): # -> chain[Unknown]:
- """ Variadic version of concat
-
- >>> list(concatv([], ["a"], ["b", "c"]))
- ['a', 'b', 'c']
-
- See also:
- itertools.chain
- """
- ...
-
-def mapcat(func, seqs): # -> chain[Unknown]:
- """ Apply func to each sequence in seqs, concatenating results.
-
- >>> list(mapcat(lambda s: [c.upper() for c in s],
- ... [["a", "b"], ["c", "d", "e"]]))
- ['A', 'B', 'C', 'D', 'E']
- """
- ...
-
-def cons(el, seq): # -> chain[Unknown]:
- """ Add el to beginning of (possibly infinite) sequence seq.
-
- >>> list(cons(1, [2, 3]))
- [1, 2, 3]
- """
- ...
-
-def interpose(el, seq): # -> chain[Unknown]:
- """ Introduce element between each pair of elements in seq
-
- >>> list(interpose("a", [1, 2, 3]))
- [1, 'a', 2, 'a', 3]
- """
- ...
-
-def frequencies(seq): # -> dict[Unknown, int]:
- """ Find number of occurrences of each value in seq
-
- >>> frequencies(['cat', 'cat', 'ox', 'pig', 'pig', 'cat']) #doctest: +SKIP
- {'cat': 3, 'ox': 1, 'pig': 2}
-
- See Also:
- countby
- groupby
- """
- ...
-
-def reduceby(key, binop, seq, init=...): # -> dict[Unknown, Unknown]:
- """ Perform a simultaneous groupby and reduction
-
- The computation:
-
- >>> result = reduceby(key, binop, seq, init) # doctest: +SKIP
-
- is equivalent to the following:
-
- >>> def reduction(group): # doctest: +SKIP
- ... return reduce(binop, group, init) # doctest: +SKIP
-
- >>> groups = groupby(key, seq) # doctest: +SKIP
- >>> result = valmap(reduction, groups) # doctest: +SKIP
-
- But the former does not build the intermediate groups, allowing it to
- operate in much less space. This makes it suitable for larger datasets
- that do not fit comfortably in memory
-
- The ``init`` keyword argument is the default initialization of the
- reduction. This can be either a constant value like ``0`` or a callable
- like ``lambda : 0`` as might be used in ``defaultdict``.
-
- Simple Examples
- ---------------
-
- >>> from operator import add, mul
- >>> iseven = lambda x: x % 2 == 0
-
- >>> data = [1, 2, 3, 4, 5]
-
- >>> reduceby(iseven, add, data) # doctest: +SKIP
- {False: 9, True: 6}
-
- >>> reduceby(iseven, mul, data) # doctest: +SKIP
- {False: 15, True: 8}
-
- Complex Example
- ---------------
-
- >>> projects = [{'name': 'build roads', 'state': 'CA', 'cost': 1000000},
- ... {'name': 'fight crime', 'state': 'IL', 'cost': 100000},
- ... {'name': 'help farmers', 'state': 'IL', 'cost': 2000000},
- ... {'name': 'help farmers', 'state': 'CA', 'cost': 200000}]
-
- >>> reduceby('state', # doctest: +SKIP
- ... lambda acc, x: acc + x['cost'],
- ... projects, 0)
- {'CA': 1200000, 'IL': 2100000}
-
- Example Using ``init``
- ----------------------
-
- >>> def set_add(s, i):
- ... s.add(i)
- ... return s
-
- >>> reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2, 3], set) # doctest: +SKIP
- {True: set([2, 4]),
- False: set([1, 3])}
- """
- ...
-
-def iterate(func, x): # -> Generator[Unknown, Any, None]:
- """ Repeatedly apply a function func onto an original input
-
- Yields x, then func(x), then func(func(x)), then func(func(func(x))), etc..
-
- >>> def inc(x): return x + 1
- >>> counter = iterate(inc, 0)
- >>> next(counter)
- 0
- >>> next(counter)
- 1
- >>> next(counter)
- 2
-
- >>> double = lambda x: x * 2
- >>> powers_of_two = iterate(double, 1)
- >>> next(powers_of_two)
- 1
- >>> next(powers_of_two)
- 2
- >>> next(powers_of_two)
- 4
- >>> next(powers_of_two)
- 8
- """
- ...
-
-def sliding_window(n, seq): # -> zip[tuple[Unknown]]:
- """ A sequence of overlapping subsequences
-
- >>> list(sliding_window(2, [1, 2, 3, 4]))
- [(1, 2), (2, 3), (3, 4)]
-
- This function creates a sliding window suitable for transformations like
- sliding means / smoothing
-
- >>> mean = lambda seq: float(sum(seq)) / len(seq)
- >>> list(map(mean, sliding_window(2, [1, 2, 3, 4])))
- [1.5, 2.5, 3.5]
- """
- ...
-
-no_pad = ...
-def partition(n, seq, pad=...): # -> zip[Unknown] | zip_longest[Unknown]:
- """ Partition sequence into tuples of length n
-
- >>> list(partition(2, [1, 2, 3, 4]))
- [(1, 2), (3, 4)]
-
- If the length of ``seq`` is not evenly divisible by ``n``, the final tuple
- is dropped if ``pad`` is not specified, or filled to length ``n`` by pad:
-
- >>> list(partition(2, [1, 2, 3, 4, 5]))
- [(1, 2), (3, 4)]
-
- >>> list(partition(2, [1, 2, 3, 4, 5], pad=None))
- [(1, 2), (3, 4), (5, None)]
-
- See Also:
- partition_all
- """
- ...
-
-def partition_all(n, seq): # -> Generator[Unknown, Any, None]:
- """ Partition all elements of sequence into tuples of length at most n
-
- The final tuple may be shorter to accommodate extra elements.
-
- >>> list(partition_all(2, [1, 2, 3, 4]))
- [(1, 2), (3, 4)]
-
- >>> list(partition_all(2, [1, 2, 3, 4, 5]))
- [(1, 2), (3, 4), (5,)]
-
- See Also:
- partition
- """
- ...
-
-def count(seq): # -> int:
- """ Count the number of items in seq
-
- Like the builtin ``len`` but works on lazy sequences.
-
- Not to be confused with ``itertools.count``
-
- See also:
- len
- """
- ...
-
-def pluck(ind, seqs, default=...): # -> map[Any] | Generator[tuple[Unknown | str, ...], None, None] | Generator[Unknown | str, None, None]:
- """ plucks an element or several elements from each item in a sequence.
-
- ``pluck`` maps ``itertoolz.get`` over a sequence and returns one or more
- elements of each item in the sequence.
-
- This is equivalent to running `map(curried.get(ind), seqs)`
-
- ``ind`` can be either a single string/index or a list of strings/indices.
- ``seqs`` should be sequence containing sequences or dicts.
-
- e.g.
-
- >>> data = [{'id': 1, 'name': 'Cheese'}, {'id': 2, 'name': 'Pies'}]
- >>> list(pluck('name', data))
- ['Cheese', 'Pies']
- >>> list(pluck([0, 1], [[1, 2, 3], [4, 5, 7]]))
- [(1, 2), (4, 5)]
-
- See Also:
- get
- map
- """
- ...
-
-def getter(index): # -> ((x: Unknown) -> tuple[Unknown]) | itemgetter[Unknown] | ((x: Unknown) -> tuple[()]):
- ...
-
-def join(leftkey, leftseq, rightkey, rightseq, left_default=..., right_default=...): # -> Generator[tuple[Unknown, Unknown] | tuple[str, Unknown] | tuple[Unknown, str], Any, None]:
- """ Join two sequences on common attributes
-
- This is a semi-streaming operation. The LEFT sequence is fully evaluated
- and placed into memory. The RIGHT sequence is evaluated lazily and so can
- be arbitrarily large.
- (Note: If right_default is defined, then unique keys of rightseq
- will also be stored in memory.)
-
- >>> friends = [('Alice', 'Edith'),
- ... ('Alice', 'Zhao'),
- ... ('Edith', 'Alice'),
- ... ('Zhao', 'Alice'),
- ... ('Zhao', 'Edith')]
-
- >>> cities = [('Alice', 'NYC'),
- ... ('Alice', 'Chicago'),
- ... ('Dan', 'Syndey'),
- ... ('Edith', 'Paris'),
- ... ('Edith', 'Berlin'),
- ... ('Zhao', 'Shanghai')]
-
- >>> # Vacation opportunities
- >>> # In what cities do people have friends?
- >>> result = join(second, friends,
- ... first, cities)
- >>> for ((a, b), (c, d)) in sorted(unique(result)):
- ... print((a, d))
- ('Alice', 'Berlin')
- ('Alice', 'Paris')
- ('Alice', 'Shanghai')
- ('Edith', 'Chicago')
- ('Edith', 'NYC')
- ('Zhao', 'Chicago')
- ('Zhao', 'NYC')
- ('Zhao', 'Berlin')
- ('Zhao', 'Paris')
-
- Specify outer joins with keyword arguments ``left_default`` and/or
- ``right_default``. Here is a full outer join in which unmatched elements
- are paired with None.
-
- >>> identity = lambda x: x
- >>> list(join(identity, [1, 2, 3],
- ... identity, [2, 3, 4],
- ... left_default=None, right_default=None))
- [(2, 2), (3, 3), (None, 4), (1, None)]
-
- Usually the key arguments are callables to be applied to the sequences. If
- the keys are not obviously callable then it is assumed that indexing was
- intended, e.g. the following is a legal change.
- The join is implemented as a hash join and the keys of leftseq must be
- hashable. Additionally, if right_default is defined, then keys of rightseq
- must also be hashable.
-
- >>> # result = join(second, friends, first, cities)
- >>> result = join(1, friends, 0, cities) # doctest: +SKIP
- """
- ...
-
-def diff(*seqs, **kwargs): # -> Generator[Unknown, Any, None]:
- """ Return those items that differ between sequences
-
- >>> list(diff([1, 2, 3], [1, 2, 10, 100]))
- [(3, 10)]
-
- Shorter sequences may be padded with a ``default`` value:
-
- >>> list(diff([1, 2, 3], [1, 2, 10, 100], default=None))
- [(3, 10), (None, 100)]
-
- A ``key`` function may also be applied to each item to use during
- comparisons:
-
- >>> list(diff(['apples', 'bananas'], ['Apples', 'Oranges'], key=str.lower))
- [('bananas', 'Oranges')]
- """
- ...
-
-def topk(k, seq, key=...): # -> tuple[Unknown, ...]:
- """ Find the k largest elements of a sequence
-
- Operates lazily in ``n*log(k)`` time
-
- >>> topk(2, [1, 100, 10, 1000])
- (1000, 100)
-
- Use a key function to change sorted order
-
- >>> topk(2, ['Alice', 'Bob', 'Charlie', 'Dan'], key=len)
- ('Charlie', 'Alice')
-
- See also:
- heapq.nlargest
- """
- ...
-
-def peek(seq): # -> tuple[Unknown, chain[Unknown]]:
- """ Retrieve the next element of a sequence
-
- Returns the first element and an iterable equivalent to the original
- sequence, still having the element retrieved.
-
- >>> seq = [0, 1, 2, 3, 4]
- >>> first, seq = peek(seq)
- >>> first
- 0
- >>> list(seq)
- [0, 1, 2, 3, 4]
- """
- ...
-
-def peekn(n, seq): # -> tuple[tuple[Unknown, ...], chain[Unknown]]:
- """ Retrieve the next n elements of a sequence
-
- Returns a tuple of the first n elements and an iterable equivalent
- to the original, still having the elements retrieved.
-
- >>> seq = [0, 1, 2, 3, 4]
- >>> first_two, seq = peekn(2, seq)
- >>> first_two
- (0, 1)
- >>> list(seq)
- [0, 1, 2, 3, 4]
- """
- ...
-
-def random_sample(prob, seq, random_state=...): # -> filter[Unknown]:
- """ Return elements from a sequence with probability of prob
-
- Returns a lazy iterator of random items from seq.
-
- ``random_sample`` considers each item independently and without
- replacement. See below how the first time it returned 13 items and the
- next time it returned 6 items.
-
- >>> seq = list(range(100))
- >>> list(random_sample(0.1, seq)) # doctest: +SKIP
- [6, 9, 19, 35, 45, 50, 58, 62, 68, 72, 78, 86, 95]
- >>> list(random_sample(0.1, seq)) # doctest: +SKIP
- [6, 44, 54, 61, 69, 94]
-
- Providing an integer seed for ``random_state`` will result in
- deterministic sampling. Given the same seed it will return the same sample
- every time.
-
- >>> list(random_sample(0.1, seq, random_state=2016))
- [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98]
- >>> list(random_sample(0.1, seq, random_state=2016))
- [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98]
-
- ``random_state`` can also be any object with a method ``random`` that
- returns floats between 0.0 and 1.0 (exclusive).
-
- >>> from random import Random
- >>> randobj = Random(2016)
- >>> list(random_sample(0.1, seq, random_state=randobj))
- [7, 9, 19, 25, 30, 32, 34, 48, 59, 60, 81, 98]
- """
- ...
-
diff --git a/typings/toolz/recipes.pyi b/typings/toolz/recipes.pyi
deleted file mode 100644
index 7723febc..00000000
--- a/typings/toolz/recipes.pyi
+++ /dev/null
@@ -1,42 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-__all__ = ('countby', 'partitionby')
-def countby(key, seq): # -> dict[Unknown, int]:
- """ Count elements of a collection by a key function
-
- >>> countby(len, ['cat', 'mouse', 'dog'])
- {3: 2, 5: 1}
-
- >>> def iseven(x): return x % 2 == 0
- >>> countby(iseven, [1, 2, 3]) # doctest:+SKIP
- {True: 1, False: 2}
-
- See Also:
- groupby
- """
- ...
-
-def partitionby(func, seq): # -> map[tuple[Unknown, ...]]:
- """ Partition a sequence according to a function
-
- Partition `s` into a sequence of lists such that, when traversing
- `s`, every time the output of `func` changes a new list is started
- and that and subsequent items are collected into that list.
-
- >>> is_space = lambda c: c == " "
- >>> list(partitionby(is_space, "I have space"))
- [('I',), (' ',), ('h', 'a', 'v', 'e'), (' ',), ('s', 'p', 'a', 'c', 'e')]
-
- >>> is_large = lambda x: x > 10
- >>> list(partitionby(is_large, [1, 2, 1, 99, 88, 33, 99, -1, 5]))
- [(1, 2, 1), (99, 88, 33, 99), (-1, 5)]
-
- See also:
- partition
- groupby
- itertools.groupby
- """
- ...
-
diff --git a/typings/toolz/sandbox/__init__.pyi b/typings/toolz/sandbox/__init__.pyi
deleted file mode 100644
index c0c0a14b..00000000
--- a/typings/toolz/sandbox/__init__.pyi
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from .core import EqualityHashKey, unzip
-from .parallel import fold
-
diff --git a/typings/toolz/sandbox/core.pyi b/typings/toolz/sandbox/core.pyi
deleted file mode 100644
index 739e0775..00000000
--- a/typings/toolz/sandbox/core.pyi
+++ /dev/null
@@ -1,104 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-class EqualityHashKey:
- """ Create a hash key that uses equality comparisons between items.
-
- This may be used to create hash keys for otherwise unhashable types:
-
- >>> from toolz import curry
- >>> EqualityHashDefault = curry(EqualityHashKey, None)
- >>> set(map(EqualityHashDefault, [[], (), [1], [1]])) # doctest: +SKIP
- {=[]=, =()=, =[1]=}
-
- **Caution:** adding N ``EqualityHashKey`` items to a hash container
- may require O(N**2) operations, not O(N) as for typical hashable types.
- Therefore, a suitable key function such as ``tuple`` or ``frozenset``
- is usually preferred over using ``EqualityHashKey`` if possible.
-
- The ``key`` argument to ``EqualityHashKey`` should be a function or
- index that returns a hashable object that effectively distinguishes
- unequal items. This helps avoid the poor scaling that occurs when
- using the default key. For example, the above example can be improved
- by using a key function that distinguishes items by length or type:
-
- >>> EqualityHashLen = curry(EqualityHashKey, len)
- >>> EqualityHashType = curry(EqualityHashKey, type) # this works too
- >>> set(map(EqualityHashLen, [[], (), [1], [1]])) # doctest: +SKIP
- {=[]=, =()=, =[1]=}
-
- ``EqualityHashKey`` is convenient to use when a suitable key function
- is complicated or unavailable. For example, the following returns all
- unique values based on equality:
-
- >>> from toolz import unique
- >>> vals = [[], [], (), [1], [1], [2], {}, {}, {}]
- >>> list(unique(vals, key=EqualityHashDefault))
- [[], (), [1], [2], {}]
-
- **Warning:** don't change the equality value of an item already in a hash
- container. Unhashable types are unhashable for a reason. For example:
-
- >>> L1 = [1] ; L2 = [2]
- >>> s = set(map(EqualityHashDefault, [L1, L2]))
- >>> s # doctest: +SKIP
- {=[1]=, =[2]=}
-
- >>> L1[0] = 2 # Don't do this! ``s`` now has duplicate items!
- >>> s # doctest: +SKIP
- {=[2]=, =[2]=}
-
- Although this may appear problematic, immutable data types is a common
- idiom in functional programming, and``EqualityHashKey`` easily allows
- the same idiom to be used by convention rather than strict requirement.
-
- See Also:
- identity
- """
- __slots__ = ...
- _default_hashkey = ...
- def __init__(self, key, item) -> None:
- ...
-
- def __hash__(self) -> int:
- ...
-
- def __eq__(self, other) -> bool:
- ...
-
- def __ne__(self, other) -> bool:
- ...
-
- def __str__(self) -> str:
- ...
-
- def __repr__(self): # -> str:
- ...
-
-
-
-def unzip(seq): # -> tuple[Unknown, ...] | tuple[map[Any] | Generator[tuple[Unknown | str, ...], None, None] | Generator[Unknown | str, None, None], ...]:
- """Inverse of ``zip``
-
- >>> a, b = unzip([('a', 1), ('b', 2)])
- >>> list(a)
- ['a', 'b']
- >>> list(b)
- [1, 2]
-
- Unlike the naive implementation ``def unzip(seq): zip(*seq)`` this
- implementation can handle an infinite sequence ``seq``.
-
- Caveats:
-
- * The implementation uses ``tee``, and so can use a significant amount
- of auxiliary storage if the resulting iterators are consumed at
- different times.
-
- * The inner sequence cannot be infinite. In Python 3 ``zip(*seq)`` can be
- used if ``seq`` is a finite sequence of infinite sequences.
-
- """
- ...
-
diff --git a/typings/toolz/sandbox/parallel.pyi b/typings/toolz/sandbox/parallel.pyi
deleted file mode 100644
index 9c872462..00000000
--- a/typings/toolz/sandbox/parallel.pyi
+++ /dev/null
@@ -1,46 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-def fold(binop, seq, default=..., map=..., chunksize=..., combine=...):
- """
- Reduce without guarantee of ordered reduction.
-
- inputs:
-
- ``binop`` - associative operator. The associative property allows us to
- leverage a parallel map to perform reductions in parallel.
- ``seq`` - a sequence to be aggregated
- ``default`` - an identity element like 0 for ``add`` or 1 for mul
-
- ``map`` - an implementation of ``map``. This may be parallel and
- determines how work is distributed.
- ``chunksize`` - Number of elements of ``seq`` that should be handled
- within a single function call
- ``combine`` - Binary operator to combine two intermediate results.
- If ``binop`` is of type (total, item) -> total
- then ``combine`` is of type (total, total) -> total
- Defaults to ``binop`` for common case of operators like add
-
- Fold chunks up the collection into blocks of size ``chunksize`` and then
- feeds each of these to calls to ``reduce``. This work is distributed
- with a call to ``map``, gathered back and then refolded to finish the
- computation. In this way ``fold`` specifies only how to chunk up data but
- leaves the distribution of this work to an externally provided ``map``
- function. This function can be sequential or rely on multithreading,
- multiprocessing, or even distributed solutions.
-
- If ``map`` intends to serialize functions it should be prepared to accept
- and serialize lambdas. Note that the standard ``pickle`` module fails
- here.
-
- Example
- -------
-
- >>> # Provide a parallel map to accomplish a parallel sum
- >>> from operator import add
- >>> fold(add, [1, 2, 3, 4], chunksize=2, map=map)
- 10
- """
- ...
-
diff --git a/typings/toolz/utils.pyi b/typings/toolz/utils.pyi
deleted file mode 100644
index cb6a3b99..00000000
--- a/typings/toolz/utils.pyi
+++ /dev/null
@@ -1,8 +0,0 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-def raises(err, lamda): # -> bool:
- ...
-
-no_default = ...
diff --git a/typings/vcr/__init__.pyi b/typings/vcr/__init__.pyi
index ba5c4e76..7838a770 100644
--- a/typings/vcr/__init__.pyi
+++ b/typings/vcr/__init__.pyi
@@ -7,6 +7,9 @@ from .config import VCR
from logging import NullHandler
from .record_mode import RecordMode as mode
+"""
+This type stub file was generated by pyright.
+"""
__version__ = ...
default_vcr = ...
use_cassette = ...
diff --git a/typings/vcr/_handle_coroutine.pyi b/typings/vcr/_handle_coroutine.pyi
index 4e7d21f0..7f3ff6e2 100644
--- a/typings/vcr/_handle_coroutine.pyi
+++ b/typings/vcr/_handle_coroutine.pyi
@@ -2,6 +2,9 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
async def handle_coroutine(vcr, fn):
...
diff --git a/typings/vcr/cassette.pyi b/typings/vcr/cassette.pyi
index a791ad96..f505a781 100644
--- a/typings/vcr/cassette.pyi
+++ b/typings/vcr/cassette.pyi
@@ -4,6 +4,9 @@ This type stub file was generated by pyright.
import wrapt
+"""
+This type stub file was generated by pyright.
+"""
log = ...
class CassetteContextDecorator:
"""Context manager/decorator that handles installing the cassette and
@@ -24,7 +27,7 @@ class CassetteContextDecorator:
"""
_non_cassette_arguments = ...
@classmethod
- def from_args(cls, cassette_class, **kwargs): # -> Self@CassetteContextDecorator:
+ def from_args(cls, cassette_class, **kwargs):
...
def __init__(self, cls, args_getter) -> None:
@@ -33,11 +36,11 @@ class CassetteContextDecorator:
def __enter__(self):
...
- def __exit__(self, *args): # -> None:
+ def __exit__(self, *args):
...
@wrapt.decorator
- def __call__(self, function, instance, args, kwargs): # -> Coroutine[Any, Any, Unknown] | Generator[Unknown, Unknown, None]:
+ def __call__(self, function, instance, args, kwargs):
...
@staticmethod
@@ -49,50 +52,50 @@ class CassetteContextDecorator:
class Cassette:
"""A container for recorded requests and responses"""
@classmethod
- def load(cls, **kwargs): # -> Self@Cassette:
+ def load(cls, **kwargs):
"""Instantiate and load the cassette stored at the specified path."""
...
@classmethod
- def use_arg_getter(cls, arg_getter): # -> CassetteContextDecorator:
+ def use_arg_getter(cls, arg_getter):
...
@classmethod
- def use(cls, **kwargs): # -> CassetteContextDecorator:
+ def use(cls, **kwargs):
...
def __init__(self, path, serializer=..., persister=..., record_mode=..., match_on=..., before_record_request=..., before_record_response=..., custom_patches=..., inject=..., allow_playback_repeats=...) -> None:
...
@property
- def play_count(self): # -> int:
+ def play_count(self):
...
@property
- def all_played(self): # -> bool:
+ def all_played(self):
"""Returns True if all responses have been played, False otherwise."""
...
@property
- def requests(self): # -> list[Unknown]:
+ def requests(self):
...
@property
- def responses(self): # -> list[Unknown]:
+ def responses(self):
...
@property
- def write_protected(self): # -> bool:
+ def write_protected(self):
...
- def append(self, request, response): # -> None:
+ def append(self, request, response):
"""Add a request, response pair to this cassette"""
...
def filter_request(self, request):
...
- def can_play_response_for(self, request): # -> bool:
+ def can_play_response_for(self, request):
...
def play_response(self, request):
@@ -102,7 +105,7 @@ class Cassette:
"""
...
- def responses_of(self, request): # -> list[Unknown]:
+ def responses_of(self, request):
"""
Find the responses corresponding to a request.
This function isn't actually used by VCR internally, but is
@@ -110,10 +113,10 @@ class Cassette:
"""
...
- def rewind(self): # -> None:
+ def rewind(self):
...
- def find_requests_with_most_matches(self, request): # -> list[Unknown]:
+ def find_requests_with_most_matches(self, request):
"""
Get the most similar request(s) stored in the cassette
of a given request as a list of tuples like this:
@@ -129,11 +132,11 @@ class Cassette:
def __str__(self) -> str:
...
- def __len__(self): # -> int:
+ def __len__(self):
"""Return the number of request,response pairs stored in here"""
...
- def __contains__(self, request): # -> bool:
+ def __contains__(self, request):
"""Return whether or not a request has been stored"""
...
diff --git a/typings/vcr/config.pyi b/typings/vcr/config.pyi
index 032606f7..9a5f7ca0 100644
--- a/typings/vcr/config.pyi
+++ b/typings/vcr/config.pyi
@@ -2,34 +2,37 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
class VCR:
@staticmethod
- def is_test_method(method_name, function): # -> bool:
+ def is_test_method(method_name, function):
...
@staticmethod
- def ensure_suffix(suffix): # -> (path: Unknown) -> Unknown:
+ def ensure_suffix(suffix):
...
def __init__(self, path_transformer=..., before_record_request=..., custom_patches=..., filter_query_parameters=..., ignore_hosts=..., record_mode=..., ignore_localhost=..., filter_headers=..., before_record_response=..., filter_post_data_parameters=..., match_on=..., before_record=..., inject_cassette=..., serializer=..., cassette_library_dir=..., func_path_generator=..., decode_compressed_response=...) -> None:
...
- def use_cassette(self, path=..., **kwargs): # -> CassetteContextDecorator:
+ def use_cassette(self, path=..., **kwargs):
...
- def get_merged_config(self, **kwargs): # -> dict[str, Module("vcr.config.yamlserializer") | Module("vcr.config.jsonserializer") | Type[FilesystemPersister] | Unknown | list[Unknown] | RecordMode | ((request: Unknown) -> Unknown) | ((response: Unknown) -> Unknown) | tuple[Unknown, ...] | bool | ((incoming: Unknown) -> Unknown) | ((function: Unknown) -> str) | None]:
+ def get_merged_config(self, **kwargs):
...
- def register_serializer(self, name, serializer): # -> None:
+ def register_serializer(self, name, serializer):
...
- def register_matcher(self, name, matcher): # -> None:
+ def register_matcher(self, name, matcher):
...
- def register_persister(self, persister): # -> None:
+ def register_persister(self, persister):
...
- def test_case(self, predicate=...): # -> type:
+ def test_case(self, predicate=...):
...
diff --git a/typings/vcr/errors.pyi b/typings/vcr/errors.pyi
index 2c0f6275..9c54c668 100644
--- a/typings/vcr/errors.pyi
+++ b/typings/vcr/errors.pyi
@@ -2,6 +2,9 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
class CannotOverwriteExistingCassetteException(Exception):
def __init__(self, *args, **kwargs) -> None:
...
diff --git a/typings/vcr/filters.pyi b/typings/vcr/filters.pyi
index c1de01d5..a2b86bdd 100644
--- a/typings/vcr/filters.pyi
+++ b/typings/vcr/filters.pyi
@@ -2,6 +2,9 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
def replace_headers(request, replacements):
"""Replace headers in request according to replacements.
The replacements should be a list of (key, value) pairs where the value can be any of:
diff --git a/typings/vcr/matchers.pyi b/typings/vcr/matchers.pyi
index 2991dc6a..0a588c6c 100644
--- a/typings/vcr/matchers.pyi
+++ b/typings/vcr/matchers.pyi
@@ -2,44 +2,47 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
log = ...
-def method(r1, r2): # -> None:
+def method(r1, r2):
...
-def uri(r1, r2): # -> None:
+def uri(r1, r2):
...
-def host(r1, r2): # -> None:
+def host(r1, r2):
...
-def scheme(r1, r2): # -> None:
+def scheme(r1, r2):
...
-def port(r1, r2): # -> None:
+def port(r1, r2):
...
-def path(r1, r2): # -> None:
+def path(r1, r2):
...
-def query(r1, r2): # -> None:
+def query(r1, r2):
...
-def raw_body(r1, r2): # -> None:
+def raw_body(r1, r2):
...
-def body(r1, r2): # -> None:
+def body(r1, r2):
...
-def headers(r1, r2): # -> None:
+def headers(r1, r2):
...
_xml_header_checker = ...
_xmlrpc_header_checker = ...
_checker_transformer_pairs = ...
-def requests_match(r1, r2, matchers): # -> bool:
+def requests_match(r1, r2, matchers):
...
-def get_matchers_results(r1, r2, matchers): # -> tuple[list[Unknown], list[Unknown]]:
+def get_matchers_results(r1, r2, matchers):
"""
Get the comparison results of two requests as two list.
The first returned list represents the matchers names that passed.
diff --git a/typings/vcr/migration.pyi b/typings/vcr/migration.pyi
index 9fff8d7f..f6b4d416 100644
--- a/typings/vcr/migration.pyi
+++ b/typings/vcr/migration.pyi
@@ -3,38 +3,28 @@ This type stub file was generated by pyright.
"""
"""
-Migration script for old 'yaml' and 'json' cassettes
-
-.. warning:: Backup your cassettes files before migration.
-
-It merges and deletes the request obsolete keys (protocol, host, port, path)
-into new 'uri' key.
-Usage::
-
- python -m vcr.migration PATH
-
-The PATH can be path to the directory with cassettes or cassette itself
+This type stub file was generated by pyright.
"""
def preprocess_yaml(cassette):
...
PARTS = ...
-def build_uri(**parts): # -> LiteralString:
+def build_uri(**parts):
...
-def migrate_json(in_fp, out_fp): # -> bool:
+def migrate_json(in_fp, out_fp):
...
-def migrate_yml(in_fp, out_fp): # -> bool:
+def migrate_yml(in_fp, out_fp):
...
-def migrate(file_path, migration_fn): # -> bool:
+def migrate(file_path, migration_fn):
...
-def try_migrate(path): # -> bool:
+def try_migrate(path):
...
-def main(): # -> None:
+def main():
...
if __name__ == "__main__":
diff --git a/typings/vcr/patch.pyi b/typings/vcr/patch.pyi
index cbc14389..36bef07d 100644
--- a/typings/vcr/patch.pyi
+++ b/typings/vcr/patch.pyi
@@ -7,7 +7,9 @@ import http.client as httplib
import urllib3.connectionpool as cpool
import requests.packages.urllib3.connectionpool as cpool
-"""Utilities for patching in cassettes"""
+"""
+This type stub file was generated by pyright.
+"""
log = ...
_HTTPConnection = httplib.HTTPConnection
_HTTPSConnection = httplib.HTTPSConnection
@@ -18,7 +20,7 @@ class CassettePatcherBuilder:
def __init__(self, cassette) -> None:
...
- def build(self): # -> chain[Any]:
+ def build(self):
...
@@ -27,24 +29,24 @@ class ConnectionRemover:
def __init__(self, connection_class) -> None:
...
- def add_connection_to_pool_entry(self, pool, connection): # -> None:
+ def add_connection_to_pool_entry(self, pool, connection):
...
- def remove_connection_to_pool_entry(self, pool, connection): # -> None:
+ def remove_connection_to_pool_entry(self, pool, connection):
...
- def __enter__(self): # -> Self@ConnectionRemover:
+ def __enter__(self):
...
- def __exit__(self, *args): # -> None:
+ def __exit__(self, *args):
...
-def reset_patchers(): # -> Generator[_patch[_HTTPConnection] | _patch[_HTTPSConnection] | _patch[Unknown | VerifiedHTTPSConnection] | _patch[Unknown | HTTPConnection] | _patch[Unknown | HTTPSConnection] | _patch[Unknown] | _patch[_HTTPConnectionWithTimeout] | _patch[_HTTPSConnectionWithTimeout], Any, None]:
+def reset_patchers():
...
@contextlib.contextmanager
-def force_reset(): # -> Generator[None, Any, None]:
+def force_reset():
...
diff --git a/typings/vcr/persisters/__init__.pyi b/typings/vcr/persisters/__init__.pyi
index 006bc274..e897159a 100644
--- a/typings/vcr/persisters/__init__.pyi
+++ b/typings/vcr/persisters/__init__.pyi
@@ -2,3 +2,6 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
diff --git a/typings/vcr/record_mode.pyi b/typings/vcr/record_mode.pyi
index 021601c2..7ff7ea05 100644
--- a/typings/vcr/record_mode.pyi
+++ b/typings/vcr/record_mode.pyi
@@ -4,6 +4,9 @@ This type stub file was generated by pyright.
from enum import Enum
+"""
+This type stub file was generated by pyright.
+"""
class RecordMode(str, Enum):
"""
Configures when VCR will record to the cassette.
diff --git a/typings/vcr/request.pyi b/typings/vcr/request.pyi
index ffba4584..9fee2578 100644
--- a/typings/vcr/request.pyi
+++ b/typings/vcr/request.pyi
@@ -4,6 +4,9 @@ This type stub file was generated by pyright.
from .util import CaseInsensitiveDict
+"""
+This type stub file was generated by pyright.
+"""
log = ...
class Request:
"""
@@ -13,56 +16,56 @@ class Request:
...
@property
- def headers(self): # -> HeadersDict:
+ def headers(self):
...
@headers.setter
- def headers(self, value): # -> None:
+ def headers(self, value):
...
@property
- def body(self): # -> BytesIO | bytes:
+ def body(self):
...
@body.setter
- def body(self, value): # -> None:
+ def body(self, value):
...
- def add_header(self, key, value): # -> None:
+ def add_header(self, key, value):
...
@property
- def scheme(self): # -> str:
+ def scheme(self):
...
@property
- def host(self): # -> str | None:
+ def host(self):
...
@property
- def port(self): # -> int | None:
+ def port(self):
...
@property
- def path(self): # -> str:
+ def path(self):
...
@property
- def query(self): # -> list[tuple[str, str]]:
+ def query(self):
...
@property
- def url(self): # -> Unknown:
+ def url(self):
...
@property
- def protocol(self): # -> str:
+ def protocol(self):
...
def __str__(self) -> str:
...
- def __repr__(self): # -> LiteralString:
+ def __repr__(self):
...
@@ -86,7 +89,7 @@ class HeadersDict(CaseInsensitiveDict):
For this reason, in cassettes I keep a dict with lists as keys, but once
deserialized into VCR, I keep them as plain, naked dicts.
"""
- def __setitem__(self, key, value): # -> None:
+ def __setitem__(self, key, value):
...
diff --git a/typings/vcr/serialize.pyi b/typings/vcr/serialize.pyi
index 78f8ca65..58b2148d 100644
--- a/typings/vcr/serialize.pyi
+++ b/typings/vcr/serialize.pyi
@@ -2,8 +2,11 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
CASSETTE_FORMAT_VERSION = ...
-def deserialize(cassette_string, serializer): # -> tuple[list[Request], list[Unknown]]:
+def deserialize(cassette_string, serializer):
...
def serialize(cassette_dict, serializer):
diff --git a/typings/vcr/serializers/__init__.pyi b/typings/vcr/serializers/__init__.pyi
index 006bc274..e897159a 100644
--- a/typings/vcr/serializers/__init__.pyi
+++ b/typings/vcr/serializers/__init__.pyi
@@ -2,3 +2,6 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
diff --git a/typings/vcr/stubs/__init__.pyi b/typings/vcr/stubs/__init__.pyi
index c69b6fb5..53712b3f 100644
--- a/typings/vcr/stubs/__init__.pyi
+++ b/typings/vcr/stubs/__init__.pyi
@@ -9,7 +9,9 @@ from vcr.request import Request
from vcr.errors import CannotOverwriteExistingCassetteException
from . import compat
-"""Stubs for patching HTTP and HTTPS requests"""
+"""
+This type stub file was generated by pyright.
+"""
log = ...
class VCRFakeSocket:
"""
@@ -17,13 +19,13 @@ class VCRFakeSocket:
Used when playing back cassettes, when there
is no actual open socket.
"""
- def close(self): # -> None:
+ def close(self):
...
- def settimeout(self, *args, **kwargs): # -> None:
+ def settimeout(self, *args, **kwargs):
...
- def fileno(self): # -> Literal[0]:
+ def fileno(self):
"""
This is kinda crappy. requests will watch
this descriptor and make sure it's not closed.
@@ -33,14 +35,14 @@ class VCRFakeSocket:
-def parse_headers(header_list): # -> HTTPMessage:
+def parse_headers(header_list):
"""
Convert headers from our serialized dict with lists for keys to a
HTTPMessage
"""
...
-def serialize_headers(response): # -> dict[Unknown, Unknown]:
+def serialize_headers(response):
...
class VCRHTTPResponse(HTTPResponse):
@@ -51,66 +53,66 @@ class VCRHTTPResponse(HTTPResponse):
...
@property
- def closed(self): # -> bool:
+ def closed(self):
...
- def read(self, *args, **kwargs): # -> bytes:
+ def read(self, *args, **kwargs):
...
def readall(self):
...
- def readinto(self, *args, **kwargs): # -> int:
+ def readinto(self, *args, **kwargs):
...
- def readline(self, *args, **kwargs): # -> bytes:
+ def readline(self, *args, **kwargs):
...
- def readlines(self, *args, **kwargs): # -> list[bytes]:
+ def readlines(self, *args, **kwargs):
...
- def seekable(self): # -> bool:
+ def seekable(self):
...
- def tell(self): # -> int:
+ def tell(self):
...
- def isatty(self): # -> bool:
+ def isatty(self):
...
- def seek(self, *args, **kwargs): # -> int:
+ def seek(self, *args, **kwargs):
...
- def close(self): # -> Literal[True]:
+ def close(self):
...
- def getcode(self): # -> int:
+ def getcode(self):
...
- def isclosed(self): # -> bool:
+ def isclosed(self):
...
- def info(self): # -> HTTPMessage:
+ def info(self):
...
- def getheaders(self): # -> list[tuple[Unknown, Unknown]]:
+ def getheaders(self):
...
- def getheader(self, header, default=...): # -> LiteralString | None:
+ def getheader(self, header, default=...):
...
- def readable(self): # -> bool:
+ def readable(self):
...
class VCRConnection:
cassette = ...
- def request(self, method, url, body=..., headers=..., *args, **kwargs): # -> None:
+ def request(self, method, url, body=..., headers=..., *args, **kwargs):
"""Persist the request metadata in self._vcr_request"""
...
- def putrequest(self, method, url, *args, **kwargs): # -> None:
+ def putrequest(self, method, url, *args, **kwargs):
"""
httplib gives you more than one way to do it. This is a way
to start building up a request. Usually followed by a bunch
@@ -118,10 +120,10 @@ class VCRConnection:
"""
...
- def putheader(self, header, *values): # -> None:
+ def putheader(self, header, *values):
...
- def send(self, data): # -> None:
+ def send(self, data):
"""
This method is called after request(), to add additional data to the
body of the request. So if that happens, let's just append the data
@@ -129,10 +131,10 @@ class VCRConnection:
"""
...
- def close(self): # -> None:
+ def close(self):
...
- def endheaders(self, message_body=...): # -> None:
+ def endheaders(self, message_body=...):
"""
Normally, this would actually send the request to the server.
We are not sending the request until getting the response,
@@ -140,14 +142,14 @@ class VCRConnection:
"""
...
- def getresponse(self, _=..., **kwargs): # -> VCRHTTPResponse:
+ def getresponse(self, _=..., **kwargs):
"""Retrieve the response"""
...
- def set_debuglevel(self, *args, **kwargs): # -> None:
+ def set_debuglevel(self, *args, **kwargs):
...
- def connect(self, *args, **kwargs): # -> Any | None:
+ def connect(self, *args, **kwargs):
"""
httplib2 uses this. Connects to the server I'm assuming.
@@ -157,17 +159,17 @@ class VCRConnection:
...
@property
- def sock(self): # -> Any | VCRFakeSocket | None:
+ def sock(self):
...
@sock.setter
- def sock(self, value): # -> None:
+ def sock(self, value):
...
def __init__(self, *args, **kwargs) -> None:
...
- def __setattr__(self, name, value): # -> None:
+ def __setattr__(self, name, value):
"""
We need to define this because any attributes that are set on the
VCRConnection need to be propagated to the real connection.
@@ -181,7 +183,7 @@ class VCRConnection:
"""
...
- def __getattr__(self, name): # -> Any:
+ def __getattr__(self, name):
"""
Send requests for weird attributes up to the real connection
(counterpart to __setattr above)
diff --git a/typings/vcr/stubs/compat.pyi b/typings/vcr/stubs/compat.pyi
index 95cde571..184fde1f 100644
--- a/typings/vcr/stubs/compat.pyi
+++ b/typings/vcr/stubs/compat.pyi
@@ -2,15 +2,18 @@
This type stub file was generated by pyright.
"""
+"""
+This type stub file was generated by pyright.
+"""
def get_header(message, name):
...
-def get_header_items(message): # -> Generator[tuple[Unknown, Unknown], Any, None]:
+def get_header_items(message):
...
-def get_headers(message): # -> Generator[tuple[Unknown, Unknown], Any, None]:
+def get_headers(message):
...
-def get_httpmessage(headers): # -> HTTPMessage:
+def get_httpmessage(headers):
...
diff --git a/typings/vcr/util.pyi b/typings/vcr/util.pyi
index d887be34..a260a984 100644
--- a/typings/vcr/util.pyi
+++ b/typings/vcr/util.pyi
@@ -4,6 +4,9 @@ This type stub file was generated by pyright.
from collections.abc import MutableMapping
+"""
+This type stub file was generated by pyright.
+"""
class CaseInsensitiveDict(MutableMapping):
"""
A case-insensitive ``dict``-like object.
@@ -29,46 +32,46 @@ class CaseInsensitiveDict(MutableMapping):
def __init__(self, data=..., **kwargs) -> None:
...
- def __setitem__(self, key, value): # -> None:
+ def __setitem__(self, key, value):
...
def __getitem__(self, key):
...
- def __delitem__(self, key): # -> None:
+ def __delitem__(self, key):
...
- def __iter__(self): # -> Generator[Unknown, None, None]:
+ def __iter__(self):
...
- def __len__(self): # -> int:
+ def __len__(self):
...
- def lower_items(self): # -> Generator[tuple[Unknown, Unknown], None, None]:
+ def lower_items(self):
"""Like iteritems(), but with all lowercase keys."""
...
def __eq__(self, other) -> bool:
...
- def copy(self): # -> CaseInsensitiveDict:
+ def copy(self):
...
- def __repr__(self): # -> str:
+ def __repr__(self):
...
-def partition_dict(predicate, dictionary): # -> tuple[dict[Unknown, Unknown], dict[Unknown, Unknown]]:
+def partition_dict(predicate, dictionary):
...
-def compose(*functions): # -> (incoming: Unknown) -> Unknown:
+def compose(*functions):
...
def read_body(request):
...
-def auto_decorate(decorator, predicate=...): # -> Type[DecorateAll]:
+def auto_decorate(decorator, predicate=...):
class DecorateAll(type):
...