diff --git a/poetry.lock b/poetry.lock index 2963d65d..bcc1423d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1116,15 +1116,13 @@ requests = ">=1.0.0" six = "*" [[package]] -name = "scrapy" +name = "Scrapy" version = "2.11.0" description = "A high-level Web Crawling and Web Scraping framework" optional = false python-versions = ">=3.8" -files = [ - {file = "Scrapy-2.11.0-py2.py3-none-any.whl", hash = "sha256:a7f36544d1f5ceb13cff9b7bc904bd7c0fc43a3af0fbe5aa2034fd937cf092d1"}, - {file = "Scrapy-2.11.0.tar.gz", hash = "sha256:3cbdedce0c3f0e0482d61be2d7458683be7cd7cf14b0ee6adfbaddb80f5b36a5"}, -] +files = [] +develop = false [package.dependencies] cryptography = ">=36.0.0" @@ -1139,13 +1137,19 @@ PyDispatcher = {version = ">=2.0.5", markers = "platform_python_implementation = pyOpenSSL = ">=21.0.0" PyPyDispatcher = {version = ">=2.1.0", markers = "platform_python_implementation == \"PyPy\""} queuelib = ">=1.4.2" -service-identity = ">=18.1.0" +service_identity = ">=18.1.0" setuptools = "*" tldextract = "*" -Twisted = ">=18.9.0,<23.8.0" +Twisted = ">=18.9.0" w3lib = ">=1.17.0" "zope.interface" = ">=5.1.0" +[package.source] +type = "git" +url = "https://github.com/scrapy/scrapy.git" +reference = "master" +resolved_reference = "c67f73069570dd6dbe8427f55d6f9e65af07132a" + [[package]] name = "scrapy-crawlera" version = "1.7.2" @@ -1288,41 +1292,39 @@ files = [ [[package]] name = "twisted" -version = "22.10.0" +version = "23.10.0" description = "An asynchronous networking framework written in Python" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8.0" files = [ - {file = "Twisted-22.10.0-py3-none-any.whl", hash = "sha256:86c55f712cc5ab6f6d64e02503352464f0400f66d4f079096d744080afcccbd0"}, - {file = "Twisted-22.10.0.tar.gz", hash = "sha256:32acbd40a94f5f46e7b42c109bfae2b302250945561783a8b7a059048f2d4d31"}, + {file = "twisted-23.10.0-py3-none-any.whl", hash = "sha256:4ae8bce12999a35f7fe6443e7f1893e6fe09588c8d2bed9c35cdce8ff2d5b444"}, + {file = "twisted-23.10.0.tar.gz", hash = "sha256:987847a0790a2c597197613686e2784fd54167df3a55d0fb17c8412305d76ce5"}, ] [package.dependencies] -attrs = ">=19.2.0" -Automat = ">=0.8.0" +attrs = ">=21.3.0" +automat = ">=0.8.0" constantly = ">=15.1" hyperlink = ">=17.1.1" -incremental = ">=21.3.0" +incremental = ">=22.10.0" twisted-iocpsupport = {version = ">=1.0.2,<2", markers = "platform_system == \"Windows\""} -typing-extensions = ">=3.6.5" -"zope.interface" = ">=4.4.2" +typing-extensions = ">=4.2.0" +zope-interface = ">=5" [package.extras] -all-non-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] -conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] -conch-nacl = ["PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] -contextvars = ["contextvars (>=2.4,<3)"] -dev = ["coverage (>=6b1,<7)", "pydoctor (>=22.9.0,<22.10.0)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=5.0,<6)", "sphinx-rtd-theme (>=1.0,<2.0)", "towncrier (>=22.8,<23.0)", "twistedchecker (>=0.7,<1.0)"] -dev-release = ["pydoctor (>=22.9.0,<22.10.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=5.0,<6)", "sphinx-rtd-theme (>=1.0,<2.0)", "towncrier (>=22.8,<23.0)"] -gtk-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pygobject", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +all-non-platform = ["twisted[conch,http2,serial,test,tls]", "twisted[conch,http2,serial,test,tls]"] +conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] +dev = ["coverage (>=6b1,<7)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "twisted[dev-release]", "twistedchecker (>=0.7,<1.0)"] +dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] +gtk-platform = ["pygobject", "pygobject", "twisted[all-non-platform]", "twisted[all-non-platform]"] http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] -mypy = ["PyHamcrest (>=1.9.0)", "PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "coverage (>=6b1,<7)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "mypy (==0.930)", "mypy-zope (==0.3.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pydoctor (>=22.9.0,<22.10.0)", "pyflakes (>=2.2,<3.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "service-identity (>=18.1.0)", "sphinx (>=5.0,<6)", "sphinx-rtd-theme (>=1.0,<2.0)", "towncrier (>=22.8,<23.0)", "twistedchecker (>=0.7,<1.0)", "types-pyOpenSSL", "types-setuptools"] -osx-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +macos-platform = ["pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "twisted[all-non-platform]", "twisted[all-non-platform]"] +mypy = ["mypy (>=1.5.1,<1.6.0)", "mypy-zope (>=1.0.1,<1.1.0)", "twisted[all-non-platform,dev]", "types-pyopenssl", "types-setuptools"] +osx-platform = ["twisted[macos-platform]", "twisted[macos-platform]"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] -test = ["PyHamcrest (>=1.9.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.0,<7.0)"] +test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] -windows-platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.0,<7.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +windows-platform = ["pywin32 (!=226)", "pywin32 (!=226)", "twisted[all-non-platform]", "twisted[all-non-platform]"] [[package]] name = "twisted-iocpsupport" @@ -1683,4 +1685,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "1d6704d65eb5af30674a08a42aa8b5f3ecb838c5a7d8bc3ae45d69b411ab0ad3" +content-hash = "0328cf35e5b39649de8631c87965f4fabd39704403a536b76a0a9961100dd75b" diff --git a/public_law/middlewares.py b/public_law/middlewares.py index b296ca4b..7970cca1 100644 --- a/public_law/middlewares.py +++ b/public_law/middlewares.py @@ -3,9 +3,8 @@ # pyright: reportUnknownVariableType=false # pyright: reportUnknownParameterType=false # pyright: reportMissingParameterType=false -# pyright: reportUnknownMemberType=false + # pyright: reportUnknownArgumentType=false -# pyright: reportGeneralTypeIssues=false # -*- coding: utf-8 -*- @@ -66,7 +65,7 @@ def process_start_requests(self, start_requests, _spider: Spider): yield req def spider_opened(self, spider: Spider): - spider.logger.info(f"Spider opened: {spider.name}") + spider.logger.info(f"Spider opened: {spider.name}") # type: ignore class OarDownloaderMiddleware: @@ -113,4 +112,4 @@ def process_exception(self, request: Request, exception, spider: Spider): pass def spider_opened(self, spider: Spider): - spider.logger.info(f"Spider opened: {spider.name}") + spider.logger.info(f"Spider opened: {spider.name}") # type: ignore diff --git a/public_law/parsers/can/doj_glossaries.py b/public_law/parsers/can/doj_glossaries.py index 5b4e25fd..c6744839 100644 --- a/public_law/parsers/can/doj_glossaries.py +++ b/public_law/parsers/can/doj_glossaries.py @@ -1,9 +1,9 @@ -# pyright: reportUnknownMemberType=false + # pyright: reportUnknownVariableType=false import re from datetime import date -from typing import Any, TypeAlias, cast +from typing import Any, TypeAlias from scrapy.http.response.html import HtmlResponse from scrapy.selector.unified import Selector, SelectorList @@ -113,7 +113,7 @@ def parse_glossary(html: HtmlResponse) -> GlossaryParseResult: entries: list[GlossaryEntry] = [] - match html.css("main dl"): + match html.selector.css("main dl"): case [first, *_]: first_dl_list = first case _: @@ -148,7 +148,7 @@ def parse_glossary(html: HtmlResponse) -> GlossaryParseResult: ) parsed_entries = tuple(entries) - url = cast(str, html.url) + url = html.url match SUBJECTS.get(url): case tuple(subjects): diff --git a/public_law/parsers/can/parliamentary_glossary.py b/public_law/parsers/can/parliamentary_glossary.py index 54964e4f..af97de76 100644 --- a/public_law/parsers/can/parliamentary_glossary.py +++ b/public_law/parsers/can/parliamentary_glossary.py @@ -15,7 +15,7 @@ def parse_glossary(html: HtmlResponse) -> GlossaryParseResult: dcterms_language="en", dcterms_coverage="CAN", # Info about original source - dcterms_source=html.url, + dcterms_source=String(html.url), publiclaw_sourceModified="unknown", publiclaw_sourceCreator=String("Parliament of Canada"), dcterms_subject=( diff --git a/public_law/parsers/usa/colorado/crs.py b/public_law/parsers/usa/colorado/crs.py index 7c4395a1..cd7e3d75 100644 --- a/public_law/parsers/usa/colorado/crs.py +++ b/public_law/parsers/usa/colorado/crs.py @@ -1,17 +1,29 @@ -# pyright: reportUnknownMemberType=false - from scrapy.selector.unified import Selector from scrapy.http.response.xml import XmlResponse -from typing import Any, Optional +from typing import Any, Optional, cast, Protocol +from toolz.functoolz import curry, flip, pipe # type: ignore +from public_law.exceptions import ParseException +from public_law.selector_util import xpath_get from public_law.text import NonemptyString, URL, titleize +import public_law.text as text from public_law.items.crs import Article, Division, Title from public_law.parsers.usa.colorado.crs_articles import parse_articles from public_law.parsers.usa.colorado.crs_divisions import parse_divisions +split = curry(flip(str.split)) +xpath_get = curry(xpath_get) + +def second(x: list[Any]) -> Any: + return x[1] + +class Logger(Protocol): + def warn(self, message: str) -> None: ... + -def parse_title_bang(dom: XmlResponse, logger: Any) -> Title: + +def parse_title_bang(dom: XmlResponse, logger: Logger) -> Title: match parse_title(dom, logger): case None: raise Exception("Could not parse title") @@ -19,46 +31,50 @@ def parse_title_bang(dom: XmlResponse, logger: Any) -> Title: return title -def parse_title(dom: XmlResponse, logger: Any) -> Optional[Title]: - match(dom.xpath("//TITLE-TEXT/text()").get()): - case str(raw_name): - name = NonemptyString(titleize(raw_name)) - case None: - logger.warn(f"Could not the parse title name in {dom.url}") - return None +def parse_title(dom: XmlResponse, logger: Logger) -> Optional[Title]: + try: + name = string_pipe( + "//TITLE-TEXT/text()", + xpath_get(dom), + titleize + ) + number = string_pipe( + "//TITLE-NUM/text()", + xpath_get(dom), + text.split_on_space, + second + ) + children = _parse_divisions_or_articles(number, dom, logger) + url = source_url(number) + return Title(name, number, children, url) - match(dom.xpath("//TITLE-NUM/text()").get()): - case str(raw_number): - number = NonemptyString(raw_number.split(" ")[1]) - case None: - logger.warn(f"Could not the parse title number in {dom.url}") - return None + except ParseException as e: + logger.warn(f"Could not parse the title: {e}") + return None - match _parse_divisions_or_articles(number, dom, logger): - case None: - return None - case children: - url_number = number.rjust(2, "0") - source_url = URL(f"https://leg.colorado.gov/sites/default/files/images/olls/crs2022-title-{url_number}.pdf") - return Title( - name = name, - number = number, - source_url = URL(source_url), - children = children - ) - - -def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | XmlResponse, logger: Any) -> Optional[list[Division] | list[Article]]: + +def string_pipe(*args: Any) -> NonemptyString: + """A wrapper around pipe() that casts the result to a NonemptyString.""" + args_with_string: Any = args + (NonemptyString,) + + return cast(NonemptyString, pipe(*args_with_string)) + + +def _parse_divisions_or_articles(title_number: NonemptyString, dom: Selector | XmlResponse, logger: Logger) -> list[Division] | list[Article]: division_nodes = dom.xpath("//T-DIV") article_nodes = dom.xpath("//TA-LIST") if len(division_nodes) > 0: - func = parse_divisions + parse_fun = parse_divisions elif len(article_nodes) > 0: - func = parse_articles + parse_fun = parse_articles else: - msg = f"Could not parse divisions or articles in Title {title_number}. Neither T-DIV nor TA-LIST nodes were found." - logger.warn(msg) - return None + msg = f"Neither T-DIV nor TA-LIST nodes were found in Title {title_number}." + raise ParseException(msg) + + return parse_fun(title_number, dom, logger) + - return func(title_number, dom, logger) +def source_url(title_number: NonemptyString) -> URL: + url_number = title_number.rjust(2, "0") + return URL(f"https://leg.colorado.gov/sites/default/files/images/olls/crs2022-title-{url_number}.pdf") diff --git a/public_law/parsers/usa/colorado/crs_articles.py b/public_law/parsers/usa/colorado/crs_articles.py index ac7d18ae..8dd39001 100644 --- a/public_law/parsers/usa/colorado/crs_articles.py +++ b/public_law/parsers/usa/colorado/crs_articles.py @@ -1,4 +1,4 @@ -# pyright: reportUnknownMemberType=false + from itertools import takewhile, dropwhile from typing import Any @@ -6,7 +6,7 @@ from bs4 import BeautifulSoup from scrapy.selector.unified import Selector -from scrapy.http.response import Response +from scrapy.http.response.xml import XmlResponse from public_law.selector_util import node_name from public_law.items.crs import * @@ -15,7 +15,7 @@ def parse_articles_from_division( title_number: NonemptyString, - dom: Selector | Response, + dom: Selector | XmlResponse, raw_div_name: str, subdiv_name: NonemptyString|None = None) -> list[Article]: @@ -25,9 +25,14 @@ def parse_articles_from_division( return _parse_articles_from_subdivision(title_number, dom, raw_div_name, subdiv_name) -def _parse_articles_from_division(title_number: NonemptyString, dom: Selector | Response, raw_div_name: str) -> list[Article]: +def _parse_articles_from_division(title_number: NonemptyString, dom_or_sel: Selector | XmlResponse, raw_div_name: str) -> list[Article]: """Return the articles within the given Division.""" + if isinstance(dom_or_sel, XmlResponse): + dom = dom_or_sel.selector + else: + dom = dom_or_sel + # # Algorithm: # @@ -61,9 +66,14 @@ def _parse_articles_from_division(title_number: NonemptyString, dom: Selector | ] -def _parse_articles_from_subdivision(title_number: NonemptyString, dom: Selector | Response, raw_div_name: str, subdiv_name: NonemptyString) -> list[Article]: +def _parse_articles_from_subdivision(title_number: NonemptyString, dom_or_sel: Selector | XmlResponse, raw_div_name: str, subdiv_name: NonemptyString) -> list[Article]: """Return the articles within the given Subdivision.""" + if isinstance(dom_or_sel, XmlResponse): + dom = dom_or_sel.selector + else: + dom = dom_or_sel + # # Algorithm: # @@ -104,7 +114,12 @@ def _parse_articles_from_subdivision(title_number: NonemptyString, dom: Selector -def parse_articles(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Article]: +def parse_articles(title_number: NonemptyString, dom_or_sel: Selector | XmlResponse, logger: Any) -> list[Article]: + if isinstance(dom_or_sel, XmlResponse): + dom = dom_or_sel.selector + else: + dom = dom_or_sel + # # Algorithm: # diff --git a/public_law/parsers/usa/colorado/crs_divisions.py b/public_law/parsers/usa/colorado/crs_divisions.py index ec8ba318..92687633 100644 --- a/public_law/parsers/usa/colorado/crs_divisions.py +++ b/public_law/parsers/usa/colorado/crs_divisions.py @@ -1,8 +1,5 @@ -# pyright: reportUnknownMemberType=false - - from scrapy.selector.unified import Selector -from scrapy.http.response import Response +from scrapy.http.response.xml import XmlResponse from typing import Any from itertools import takewhile, dropwhile @@ -15,7 +12,12 @@ -def parse_divisions(title_number: NonemptyString, dom: Selector | Response, logger: Any) -> list[Division]: +def parse_divisions(title_number: NonemptyString, dom_or_sel: Selector | XmlResponse, logger: Any) -> list[Division]: + if isinstance(dom_or_sel, XmlResponse): + dom = dom_or_sel.selector + else: + dom = dom_or_sel + division_nodes = dom.xpath("//T-DIV") divs: list[Division] = [] @@ -38,9 +40,14 @@ def parse_divisions(title_number: NonemptyString, dom: Selector | Response, logg return divs -def parse_subdivisions_from_division(title_number: NonemptyString, dom: Selector | Response, raw_div_name: str) -> list[Subdivision]: +def parse_subdivisions_from_division(title_number: NonemptyString, dom_or_sel: Selector | XmlResponse, raw_div_name: str) -> list[Subdivision]: """Return the Subdivisions within the given Division.""" + if isinstance(dom_or_sel, XmlResponse): + dom = dom_or_sel.selector + else: + dom = dom_or_sel + # # Algorithm: # @@ -78,7 +85,7 @@ def _is_subdiv_node(node: Selector) -> bool: return Subdivision.is_valid_raw_name(just_text(node)) -# def _has_subdivisions(dom: Selector | Response) -> bool: +# def _has_subdivisions(dom: Selector | XmlResponse) -> bool: # raw_div_names = [just_text(e) for e in dom.xpath("//TITLE-ANAL/T-DIV")] # return not all([Division.is_valid_raw_name(n) for n in raw_div_names]) diff --git a/public_law/parsers/usa/colorado/crs_sections.py b/public_law/parsers/usa/colorado/crs_sections.py index 723c6c0c..582d8efb 100644 --- a/public_law/parsers/usa/colorado/crs_sections.py +++ b/public_law/parsers/usa/colorado/crs_sections.py @@ -1,4 +1,4 @@ -# pyright: reportUnknownMemberType=false + from typing import Any @@ -13,7 +13,7 @@ def parse_sections(dom: XmlResponse, logger: Any) -> list[Section]: - section_nodes = dom.xpath("//SECTION-TEXT") + section_nodes = dom.selector.xpath("//SECTION-TEXT") sections: list[Section] = [] for node in section_nodes: @@ -35,7 +35,6 @@ def parse_sections(dom: XmlResponse, logger: Any) -> list[Section]: logger.warn(f"Could not parse section text for {normalize_whitespace(node.get())} in {dom.url}") continue - sections.append(Section( name = NonemptyString(name), number = NonemptyString(number), diff --git a/public_law/parsers/usa/georgia_ag_opinions.py b/public_law/parsers/usa/georgia_ag_opinions.py index d85eec67..007bf9cf 100644 --- a/public_law/parsers/usa/georgia_ag_opinions.py +++ b/public_law/parsers/usa/georgia_ag_opinions.py @@ -37,9 +37,9 @@ class OpinionParseResult(NamedTuple): def parse_ag_opinion(html: Response) -> OpinionParseResult: - summary = first(html, css=".page-top__subtitle--re p::text", expected="summary") - title = first(html, css="h1.page-top__title--opinion::text", expected="title") - date = first(html, css="time::text", expected="date") + summary = first(html, css=".page-top__subtitle--re p::text", expected="summary") + title = first(html, css="h1.page-top__title--opinion::text", expected="title") + date = first(html, css="time::text", expected="date") full_text = cast( str, pipe( @@ -71,11 +71,11 @@ def opinion_date_to_iso8601(date: str) -> str: def get_all(node: Union[Response, Selector], css: str) -> List[str]: - return node.css(css).getall() # type: ignore + return node.css(css).getall() def first(node: Response | Selector, css: str, expected: str) -> str: - match node.css(css).get(): # type: ignore + match node.css(css).get(): case str(result): return result case _: diff --git a/public_law/selector_util.py b/public_law/selector_util.py index 65783a9a..7d88bf1d 100644 --- a/public_law/selector_util.py +++ b/public_law/selector_util.py @@ -1,7 +1,10 @@ + from typing import Any from scrapy.selector.unified import Selector, SelectorList +from scrapy.http.response.xml import XmlResponse +from .exceptions import ParseException def node_name(node: Selector) -> str | None: return node.xpath("name()").get() @@ -9,3 +12,11 @@ def node_name(node: Selector) -> str | None: def just_text(node: Selector | SelectorList | Any) -> str | None: return node.xpath("text()").get() + + +def xpath_get(dom: XmlResponse, xpath: str) -> str: + match dom.xpath(xpath).get(): + case str(value): + return value + case None: + raise ParseException(f"Could not find {xpath} in {dom.url}") diff --git a/public_law/spiders/int/rome_statute.py b/public_law/spiders/int/rome_statute.py index dada58af..662521bc 100644 --- a/public_law/spiders/int/rome_statute.py +++ b/public_law/spiders/int/rome_statute.py @@ -1,9 +1,10 @@ import re -from typing import Any, Dict +from typing import Any, Dict, cast from public_law.parsers.int.rome_statute import articles, footnotes, new_metadata, parts from scrapy import Spider from scrapy.http.response import Response +from scrapy.http.response.html import HtmlResponse JD_VERBOSE_NAME = "Intergovernmental" @@ -19,7 +20,7 @@ class RomeStatute(Spider): def parse(self, response: Response, **_kwargs: Dict[str, Any]): """Scrapy framework callback which begins the parsing.""" - for url in start_page_urls(response): + for url in start_page_urls(cast(HtmlResponse, response)): if "Rome-Statute.pdf" not in url: # Skip non-English versions for now. continue @@ -40,9 +41,9 @@ def parse(self, response: Response, **_kwargs: Dict[str, Any]): # -def start_page_urls(response: Response) -> list[str]: - anchors = response.css("h2#coreICCtexts + p + div").css("a").getall()[:4] # type: ignore - relative_urls = [re.findall(r'"(.+)"', a)[0] for a in anchors] # type: ignore +def start_page_urls(response: HtmlResponse) -> list[str]: + anchors = response.css("h2#coreICCtexts + p + div").css("a").getall()[:4] + relative_urls = [re.findall(r'"(.+)"', a)[0] for a in anchors] absolute_urls = ["https://www.icc-cpi.int" + url for url in relative_urls] return absolute_urls diff --git a/public_law/spiders/usa/colorado_crs.py b/public_law/spiders/usa/colorado_crs.py index c960ae82..3f4ac7f8 100644 --- a/public_law/spiders/usa/colorado_crs.py +++ b/public_law/spiders/usa/colorado_crs.py @@ -4,7 +4,7 @@ import os from pathlib import Path -from typing import Any +from typing import Any, cast from progressbar import ProgressBar from scrapy import Spider @@ -27,9 +27,9 @@ class ColoradoCRS(Spider): def start_requests(self): """Read the files from a local directory.""" try: - dir = self.crsdata_dir + dir = cast(str, self.crsdata_dir) except: - raise Exception("No crsdata_dir specified.") + raise Exception("No crsdata_dir specified with the -a command line option.") DIR = f"{os.getcwd()}/{dir}" XML_DIR = f"{DIR}/TITLES" diff --git a/public_law/spiders/usa/georgia_ag_opinions.py b/public_law/spiders/usa/georgia_ag_opinions.py index 2f63102e..e412812f 100644 --- a/public_law/spiders/usa/georgia_ag_opinions.py +++ b/public_law/spiders/usa/georgia_ag_opinions.py @@ -1,7 +1,7 @@ # pyright: reportUninitializedInstanceVariable=false # pyright: reportPrivateUsage=false # pyright: reportUnknownVariableType=false -# pyright: reportUnknownMemberType=false + # pyright: reportUnknownArgumentType=false # pyright: reportGeneralTypeIssues=false diff --git a/public_law/spiders/usa/oregon_regs.py b/public_law/spiders/usa/oregon_regs.py index 8c22b9fa..a120171d 100644 --- a/public_law/spiders/usa/oregon_regs.py +++ b/public_law/spiders/usa/oregon_regs.py @@ -14,14 +14,15 @@ from public_law.text import titleize -JD_VERBOSE_NAME = "USA / Oregon" +JD_VERBOSE_NAME = "USA / Oregon" PUBLICATION_NAME = "Oregon Administrative Rules" class OregonRegs(Spider): - name = "usa_or_regs" + name = "usa_or_regs" allowed_domains = [DOMAIN] - start_urls = [oar_url("ruleSearch.action")] + start_urls = [oar_url("ruleSearch.action")] + def __init__(self, *args: List[str], **kwargs: Dict[str, Any]): super().__init__(*args, **kwargs) # type: ignore @@ -47,19 +48,19 @@ def parse_search_page(self, response: Response): The search page contains a list of Chapters, with the names, numbers, and internal id's. """ - for option in response.css("#browseForm option"): # type: ignore - db_id: Any = option.xpath("@value").get() # type: ignore + for option in response.css("#browseForm option"): + db_id: Any = option.xpath("@value").get() if db_id == "-1": # Ignore the heading continue - number, name = map(str.strip, option.xpath("text()").get().split("-", 1)) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] + number, name = map(str.strip, option.xpath("text()").get().split("-", 1)) # type: ignore # FIXME chapter = new_chapter(db_id, number, name) new_chapter_index = len(self.oar["chapters"]) # type: ignore - self.oar["chapters"].append(chapter) # type: ignore + self.oar["chapters"].append(chapter) # type: ignore request = Request(chapter["url"], callback=self.parse_chapter_page) # pyright: ignore[reportUnknownArgumentType] - request.meta["chapter_index"] = new_chapter_index # type: ignore + request.meta["chapter_index"] = new_chapter_index yield request def parse_chapter_page(self, response: Response): @@ -68,32 +69,32 @@ def parse_chapter_page(self, response: Response): A Chapter's page contains a hierarchical list of all its Divisions along with their contained Rules. """ - chapter: Chapter = cast(Chapter, self.oar["chapters"][response.meta["chapter_index"]]) # pyright: ignore[reportUnknownMemberType] + chapter: Chapter = cast(Chapter, self.oar["chapters"][response.meta["chapter_index"]]) # Collect the Divisions anchor: Selector for anchor in response.css("#accordion > h3 > a"): # type: ignore - db_id: str = cast(str, anchor.xpath("@href").get().split("selectedDivision=")[1]) # pyright: ignore[reportUnknownMemberType, reportOptionalMemberAccess] + db_id = anchor.xpath("@href").get().split("selectedDivision=")[1] # pyright: ignore[reportOptionalMemberAccess] raw_number, raw_name = map( - str.strip, anchor.xpath("text()").get().split("-", 1) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType, reportOptionalMemberAccess] + str.strip, anchor.xpath("text()").get().split("-", 1) # pyright: ignore[reportOptionalMemberAccess] ) number: str = raw_number.split(" ")[1] name: str = titleize(raw_name) division = new_division(db_id, number, name) - chapter["divisions"].append(division) # pyright: ignore[reportUnknownMemberType] + chapter["divisions"].append(division) # type: ignore # Request a scrape of the Division page request = Request(division["url"], callback=self.parse_division_page) # type: ignore request.meta["division_index"] = len(chapter["divisions"]) - 1 # type: ignore - request.meta["chapter_index"] = response.meta["chapter_index"] # type: ignore + request.meta["chapter_index"] = response.meta["chapter_index"] yield request def parse_division_page(self, response: Response): chapter: Chapter = self.oar["chapters"][response.meta["chapter_index"]] # type: ignore division: Division = chapter["divisions"][response.meta["division_index"]] # type: ignore - division["rules"].extend(parse_division(response)) # type: ignore + division["rules"].extend(parse_division(response)) # type: ignore # # Output a single object: a JSON tree containing all the scraped data. This @@ -118,12 +119,12 @@ def spider_idle(self, spider: Spider): # submit data _without_ also triggering a scrape. So I provide a URL # to a simple site that we're going to ignore. null_request = Request( - "https://www.public.law/about-us", callback=self.submit_data # type: ignore + "https://www.public.law/about-us", callback=self.submit_data ) self.crawler.engine.schedule(null_request, spider) # type: ignore raise scrapy.exceptions.DontCloseSpider - def submit_data(self, _): + def submit_data(self, _: Any): """Simply return the collection of all the scraped data. Ignore the actual scraped content. I haven't figured out another diff --git a/public_law/text.py b/public_law/text.py index 1e9a43d7..f661673b 100644 --- a/public_law/text.py +++ b/public_law/text.py @@ -230,3 +230,7 @@ def remove_trailing_period(text: str) -> str: if text.endswith("."): return text[:-1] return text + + +def split_on_space(s: str) -> list[str]: + return s.split(" ") diff --git a/pyproject.toml b/pyproject.toml index 5028fd08..8b9dd5bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ progressbar2 = "*" pydantic = "^2.5" python = "^3.12" pytz = "*" -scrapy = "^2.6" +scrapy = {git = "https://github.com/scrapy/scrapy.git", branch = "master"} scrapy-crawlera = "*" tika = "^1.24" titlecase = "*" @@ -41,12 +41,13 @@ typeCheckingMode = "strict" reportCallInDefaultInitializer = "error" reportImplicitStringConcatenation = "error" reportMissingSuperCall = "error" -reportMissingTypeStubs = false reportPropertyTypeMismatch = "error" reportUninitializedInstanceVariable = "error" reportUnnecessaryTypeIgnoreComment = "error" reportUnusedCallResult = "error" -reportUnusedImport = "warning" + +reportUnusedImport = false +reportMissingTypeStubs = false # I don't know the purpose. Scrapy does better w/out type stubs. [tool.pytest.ini_options] diff --git a/tests/public_law/parsers/can/doj_glossaries_test.py b/tests/public_law/parsers/can/doj_glossaries_test.py index d9bc416d..4bf2200d 100644 --- a/tests/public_law/parsers/can/doj_glossaries_test.py +++ b/tests/public_law/parsers/can/doj_glossaries_test.py @@ -1,6 +1,6 @@ # pyright: reportUnknownParameterType=false # pyright: reportMissingParameterType=false -# pyright: reportUnknownMemberType=false + # pyright: reportUnknownArgumentType=false # pyright: reportUnknownVariableType=false @@ -36,9 +36,7 @@ def p11() -> GlossaryParseResult: ) -@pytest.fixture -def glos() -> GlossaryParseResult: - return glossary_fixture( +GLOS = glossary_fixture( "glos.html", "https://www.justice.gc.ca/eng/rp-pr/fl-lf/famil/2003_5/glos.html", parse_glossary, @@ -69,13 +67,13 @@ def p18() -> GlossaryParseResult: class TestDctermsTitle: - def test_when_it_contains_an_anchor(self, glos): + def test_when_it_contains_an_anchor(self): assert ( - glos.metadata.dcterms_title + GLOS.metadata.dcterms_title == "Glossary - Managing Contact Difficulties: A Child-Centred Approach (2003-FCY-5E)" ) - def test_when_there_is_just_an_h1(self, index): + def test_when_there_is_just_an_h1(self, index: GlossaryParseResult): assert index.metadata.dcterms_title == "Glossary" # Unfortunately. def test_all_caps_title_correctly_formatted(self, p18: GlossaryParseResult): @@ -84,7 +82,7 @@ def test_all_caps_title_correctly_formatted(self, p18: GlossaryParseResult): == "Glossary of Terms - Spousal Support Advisory Guidelines July 2008" ) - def test_the_title(self, p7g): + def test_the_title(self, p7g: GlossaryParseResult): assert ( p7g.metadata.dcterms_title == "Glossary of Legal Terms - Legal Aid Program Evaluation" @@ -110,8 +108,8 @@ def test_subject_p11(self, p11): ), ) - def test_subject_glos(self, glos): - assert subj_strings(glos) == ( + def test_subject_glos(self): + assert subj_strings(GLOS) == ( ( "http://id.loc.gov/authorities/subjects/sh98001029", "Parental alienation syndrome", @@ -123,11 +121,11 @@ def test_subject_glos(self, glos): ) -def test_phrase_does_not_end_with_colon(glos): - assert first(glos.entries).phrase == "Alienated Parent" +def test_phrase_does_not_end_with_colon(): + assert first(GLOS.entries).phrase == "Alienated Parent" -def test_parses_emphasized_text(p11): +def test_parses_emphasized_text(p11: GlossaryParseResult): definition_with_em = first(p11.entries).definition expected_definition = ( "Legal term previously used in the Divorce Act to " @@ -167,37 +165,37 @@ def test_parse_error_is_fixed_4(p18: GlossaryParseResult): assert entry.definition[-4:] == "

" -def test_the_url(p7g): +def test_the_url(p7g: GlossaryParseResult): assert ( p7g.metadata.dcterms_source == "https://www.justice.gc.ca/eng/rp-pr/cp-pm/eval/rep-rap/12/lap-paj/p7g.html" ) -def test_the_author(p7g): +def test_the_author(p7g: GlossaryParseResult): assert p7g.metadata.dcterms_creator == "https://public.law" -def test_the_scrape_date(p7g): +def test_the_scrape_date(p7g: GlossaryParseResult): assert p7g.metadata.publiclaw_sourceModified == date(2022, 5, 13) -def test_the_original_modification_date(p7g): +def test_the_original_modification_date(p7g: GlossaryParseResult): assert p7g.metadata.dcterms_modified == today() -def test_proper_number_of_entries(p7g): +def test_proper_number_of_entries(p7g: GlossaryParseResult): assert len(tuple(p7g.entries)) == 36 -def test_a_term_case_1(p7g): +def test_a_term_case_1(p7g: GlossaryParseResult): entry = cast(GlossaryEntry, nth(p7g.entries, 2)) assert entry.phrase == "Adjournment" assert entry.definition == "Postponement of a court hearing to another date." -def subj_strings(glossary): +def subj_strings(glossary: GlossaryParseResult): """ Test helper: return the strings in a Glossary's subjects. """ diff --git a/tests/public_law/parsers/int/rome_statute_test.py b/tests/public_law/parsers/int/rome_statute_test.py index eea086ba..66f424b3 100644 --- a/tests/public_law/parsers/int/rome_statute_test.py +++ b/tests/public_law/parsers/int/rome_statute_test.py @@ -1,7 +1,7 @@ # # pyright: reportUninitializedInstanceVariable=false # # pyright: reportPrivateUsage=false # # pyright: reportUnknownVariableType=false -# # pyright: reportUnknownMemberType=false +# # # pyright: reportGeneralTypeIssues=false # # pyright: reportUntypedFunctionDecorator=false diff --git a/typings/progressbar/__about__.pyi b/typings/progressbar/__about__.pyi index 2902014a..987cdcd0 100644 --- a/typings/progressbar/__about__.pyi +++ b/typings/progressbar/__about__.pyi @@ -2,18 +2,9 @@ This type stub file was generated by pyright. """ -'''Text progress bar library for Python. - -A text progress bar is typically used to display the progress of a long -running operation, providing a visual cue that processing is underway. - -The ProgressBar class manages the current progress, and the format of the line -is given by a number of widgets. A widget is an object that may display -differently depending on the state of the progress bar. - -The progressbar module is very easy to use, yet very powerful. It will also -automatically enable features like auto-resizing when the system supports it. -''' +""" +This type stub file was generated by pyright. +""" __title__ = ... __package_name__ = ... __author__ = ... diff --git a/typings/progressbar/__init__.pyi b/typings/progressbar/__init__.pyi index 4f7f989c..87ed6eb4 100644 --- a/typings/progressbar/__init__.pyi +++ b/typings/progressbar/__init__.pyi @@ -10,5 +10,8 @@ from .shortcuts import progressbar from .utils import len_color, streams from .widgets import AbsoluteETA, AdaptiveETA, AdaptiveTransferSpeed, AnimatedMarker, Bar, BouncingBar, Counter, CurrentTime, DataSize, DynamicMessage, ETA, FileTransferSpeed, FormatCustomText, FormatLabel, FormatLabelBar, GranularBar, MultiProgressBar, MultiRangeBar, Percentage, PercentageLabelBar, ReverseBar, RotatingMarker, SimpleProgress, Timer, Variable, VariableMixin +""" +This type stub file was generated by pyright. +""" __date__ = ... __all__ = ['progressbar', 'len_color', 'streams', 'Timer', 'ETA', 'AdaptiveETA', 'AbsoluteETA', 'DataSize', 'FileTransferSpeed', 'AdaptiveTransferSpeed', 'AnimatedMarker', 'Counter', 'Percentage', 'FormatLabel', 'SimpleProgress', 'Bar', 'ReverseBar', 'BouncingBar', 'UnknownLength', 'ProgressBar', 'DataTransferBar', 'RotatingMarker', 'VariableMixin', 'MultiRangeBar', 'MultiProgressBar', 'GranularBar', 'FormatLabelBar', 'PercentageLabelBar', 'Variable', 'DynamicMessage', 'FormatCustomText', 'CurrentTime', 'NullBar', '__author__', '__version__'] diff --git a/typings/progressbar/bar.pyi b/typings/progressbar/bar.pyi index bf4792d4..b01e4666 100644 --- a/typings/progressbar/bar.pyi +++ b/typings/progressbar/bar.pyi @@ -7,22 +7,25 @@ from python_utils import types from collections import abc from . import base +""" +This type stub file was generated by pyright. +""" logger = ... T = types.TypeVar('T') class ProgressBarMixinBase: def __init__(self, **kwargs) -> None: ... - def start(self, **kwargs): # -> None: + def start(self, **kwargs): ... - def update(self, value=...): # -> None: + def update(self, value=...): ... - def finish(self): # -> None: + def finish(self): ... - def __del__(self): # -> None: + def __del__(self): ... @@ -35,10 +38,10 @@ class DefaultFdMixin(ProgressBarMixinBase): def __init__(self, fd: types.IO = ..., is_terminal: bool | None = ..., line_breaks: bool | None = ..., enable_colors: bool | None = ..., **kwargs) -> None: ... - def update(self, *args, **kwargs): # -> None: + def update(self, *args, **kwargs): ... - def finish(self, *args, **kwargs): # -> None: + def finish(self, *args, **kwargs): ... @@ -47,7 +50,7 @@ class ResizableMixin(ProgressBarMixinBase): def __init__(self, term_width: int | None = ..., **kwargs) -> None: ... - def finish(self): # -> None: + def finish(self): ... @@ -56,13 +59,13 @@ class StdRedirectMixin(DefaultFdMixin): def __init__(self, redirect_stderr: bool = ..., redirect_stdout: bool = ..., **kwargs) -> None: ... - def start(self, *args, **kwargs): # -> None: + def start(self, *args, **kwargs): ... - def update(self, value: float = ...): # -> None: + def update(self, value: float = ...): ... - def finish(self, end=...): # -> None: + def finish(self, end=...): ... @@ -151,14 +154,14 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ... @property - def dynamic_messages(self): # -> AttributeDict: + def dynamic_messages(self): ... @dynamic_messages.setter - def dynamic_messages(self, value): # -> None: + def dynamic_messages(self, value): ... - def init(self): # -> None: + def init(self): ''' (re)initialize values to original state so the progressbar can be used (again) @@ -166,7 +169,7 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ... @property - def percentage(self): # -> float | None: + def percentage(self): '''Return current percentage, returns None if no max_value is given >>> progress = ProgressBar() @@ -199,14 +202,14 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ''' ... - def get_last_update_time(self): # -> datetime | None: + def get_last_update_time(self): ... - def set_last_update_time(self, value): # -> None: + def set_last_update_time(self, value): ... last_update_time = ... - def data(self): # -> dict[str, T]: + def data(self): ''' Returns: @@ -236,38 +239,38 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ''' ... - def default_widgets(self): # -> list[Percentage | str | SimpleProgress | Bar | Timer | AdaptiveETA] | list[AnimatedMarker | str | BouncingBar | Counter | Timer]: + def default_widgets(self): ... - def __call__(self, iterable, max_value=...): # -> Self@ProgressBar: + def __call__(self, iterable, max_value=...): 'Use a ProgressBar to iterate through an iterable' ... - def __iter__(self): # -> Self@ProgressBar: + def __iter__(self): ... def __next__(self): ... - def __exit__(self, exc_type, exc_value, traceback): # -> None: + def __exit__(self, exc_type, exc_value, traceback): ... - def __enter__(self): # -> Self@ProgressBar: + def __enter__(self): ... next = ... - def __iadd__(self, value): # -> Self@ProgressBar: + def __iadd__(self, value): 'Updates the ProgressBar by adding a new value.' ... - def increment(self, value=..., *args, **kwargs): # -> Self@ProgressBar: + def increment(self, value=..., *args, **kwargs): ... - def update(self, value=..., force=..., **kwargs): # -> None: + def update(self, value=..., force=..., **kwargs): 'Updates the ProgressBar to a new value.' ... - def start(self, max_value=..., init=...): # -> Self@ProgressBar: + def start(self, max_value=..., init=...): '''Starts measuring time, and prints the bar at 0%. It returns self so you can use it like this: @@ -287,7 +290,7 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ''' ... - def finish(self, end=..., dirty=...): # -> None: + def finish(self, end=..., dirty=...): ''' Puts the ProgressBar bar in the finished state. @@ -303,7 +306,7 @@ class ProgressBar(StdRedirectMixin, ResizableMixin, ProgressBarBase): ... @property - def currval(self): # -> T: + def currval(self): ''' Legacy method to make progressbar-2 compatible with the original progressbar package @@ -317,7 +320,7 @@ class DataTransferBar(ProgressBar): This assumes that the values its given are numbers of bytes. ''' - def default_widgets(self): # -> list[Percentage | str | DataSize | Bar | Timer | AdaptiveETA] | list[AnimatedMarker | str | DataSize | Timer]: + def default_widgets(self): ... @@ -327,13 +330,13 @@ class NullBar(ProgressBar): Progress bar that does absolutely nothing. Useful for single verbosity flags ''' - def start(self, *args, **kwargs): # -> Self@NullBar: + def start(self, *args, **kwargs): ... - def update(self, *args, **kwargs): # -> Self@NullBar: + def update(self, *args, **kwargs): ... - def finish(self, *args, **kwargs): # -> Self@NullBar: + def finish(self, *args, **kwargs): ... diff --git a/typings/progressbar/base.pyi b/typings/progressbar/base.pyi index 5db8d135..cc55a8c5 100644 --- a/typings/progressbar/base.pyi +++ b/typings/progressbar/base.pyi @@ -2,11 +2,14 @@ This type stub file was generated by pyright. """ +""" +This type stub file was generated by pyright. +""" class FalseMeta(type): - def __bool__(self): # -> Literal[False]: + def __bool__(self): ... - def __cmp__(self, other): # -> Literal[-1]: + def __cmp__(self, other): ... __nonzero__ = ... diff --git a/typings/progressbar/shortcuts.pyi b/typings/progressbar/shortcuts.pyi index facf67b0..f10486db 100644 --- a/typings/progressbar/shortcuts.pyi +++ b/typings/progressbar/shortcuts.pyi @@ -2,6 +2,9 @@ This type stub file was generated by pyright. """ -def progressbar(iterator, min_value=..., max_value=..., widgets=..., prefix=..., suffix=..., **kwargs): # -> Generator[Unknown, Any, None]: +""" +This type stub file was generated by pyright. +""" +def progressbar(iterator, min_value=..., max_value=..., widgets=..., prefix=..., suffix=..., **kwargs): ... diff --git a/typings/progressbar/utils.pyi b/typings/progressbar/utils.pyi index fe148bd3..7c9d2c5a 100644 --- a/typings/progressbar/utils.pyi +++ b/typings/progressbar/utils.pyi @@ -5,6 +5,9 @@ This type stub file was generated by pyright. from python_utils import types from .bar import ProgressBar +""" +This type stub file was generated by pyright. +""" if types.TYPE_CHECKING: ... ANSI_TERMS = ... @@ -82,7 +85,7 @@ class WrappingIO: def __init__(self, target: types.IO, capturing: bool = ..., listeners: types.Set[ProgressBar] = ...) -> None: ... - def __getattr__(self, name): # -> Any: + def __getattr__(self, name): ... def write(self, value: str) -> None: @@ -140,7 +143,7 @@ class StreamWrapper: def flush(self) -> None: ... - def excepthook(self, exc_type, exc_value, exc_traceback): # -> None: + def excepthook(self, exc_type, exc_value, exc_traceback): ... diff --git a/typings/progressbar/widgets.pyi b/typings/progressbar/widgets.pyi index c26e98cd..397e26aa 100644 --- a/typings/progressbar/widgets.pyi +++ b/typings/progressbar/widgets.pyi @@ -6,15 +6,18 @@ import abc from python_utils import types from .bar import ProgressBar +""" +This type stub file was generated by pyright. +""" if types.TYPE_CHECKING: ... MAX_DATE = ... MAX_TIME = ... MAX_DATETIME = ... -def string_or_lambda(input_): # -> (progress: Unknown, data: Unknown, width: Unknown) -> Unknown: +def string_or_lambda(input_): ... -def create_wrapper(wrapper): # -> LiteralString | None: +def create_wrapper(wrapper): '''Convert a wrapper tuple or format string to a format string >>> create_wrapper('') @@ -27,14 +30,14 @@ def create_wrapper(wrapper): # -> LiteralString | None: ''' ... -def wrapper(function, wrapper): # -> _Wrapped[(...), Unknown, (*args: Unknown, **kwargs: Unknown), str]: +def wrapper(function, wrapper): '''Wrap the output of a function in a template string or a tuple with begin/end strings ''' ... -def create_marker(marker, wrap=...): # -> ((progress: Unknown, data: Unknown, width: Unknown) -> (Unknown | str)) | _Wrapped[(...), Unknown, (*args: Unknown, **kwargs: Unknown), str]: +def create_marker(marker, wrap=...): ... class FormatWidgetMixin: @@ -56,7 +59,7 @@ class FormatWidgetMixin: def __init__(self, format, new_style=..., **kwargs) -> None: ... - def get_format(self, progress, data, format=...): # -> Unknown: + def get_format(self, progress, data, format=...): ... def __call__(self, progress, data, format=...): @@ -92,7 +95,7 @@ class WidthWidgetMixin: def __init__(self, min_width=..., max_width=..., **kwargs) -> None: ... - def check_size(self, progress: ProgressBar): # -> bool: + def check_size(self, progress: ProgressBar): ... @@ -101,7 +104,7 @@ class WidgetBase(WidthWidgetMixin): __metaclass__ = abc.ABCMeta copy = ... @abc.abstractmethod - def __call__(self, progress, data): # -> None: + def __call__(self, progress, data): '''Updates the widget. progress - a reference to the calling ProgressBar @@ -118,7 +121,7 @@ class AutoWidthWidgetBase(WidgetBase): all have the same width, and together will fill the line. ''' @abc.abstractmethod - def __call__(self, progress, data, width): # -> None: + def __call__(self, progress, data, width): '''Updates the widget providing the total width the widget must fill. progress - a reference to the calling ProgressBar @@ -205,7 +208,7 @@ class SamplesMixin(TimeSensitiveWidgetBase): def get_sample_values(self, progress, data): ... - def __call__(self, progress, data, delta=...): # -> tuple[Unknown, Unknown] | tuple[None, None]: + def __call__(self, progress, data, delta=...): ... @@ -288,7 +291,7 @@ class AnimatedMarker(TimeSensitiveWidgetBase): def __init__(self, markers=..., default=..., fill=..., marker_wrap=..., fill_wrap=..., **kwargs) -> None: ... - def __call__(self, progress, data, width=...): # -> str: + def __call__(self, progress, data, width=...): '''Updates the widget to show the next marker or the first marker when finished''' ... @@ -311,7 +314,7 @@ class Percentage(FormatWidgetMixin, WidgetBase): def __init__(self, format=..., na=..., **kwargs) -> None: ... - def get_format(self, progress, data, format=...): # -> str | Unknown: + def get_format(self, progress, data, format=...): ... @@ -342,7 +345,7 @@ class Bar(AutoWidthWidgetBase): ''' ... - def __call__(self, progress, data, width): # -> str: + def __call__(self, progress, data, width): '''Updates the progress bar and its subcomponents''' ... @@ -366,7 +369,7 @@ class ReverseBar(Bar): class BouncingBar(Bar, TimeSensitiveWidgetBase): '''A bar which has a marker which bounces from side to side.''' INTERVAL = ... - def __call__(self, progress, data, width): # -> str: + def __call__(self, progress, data, width): '''Updates the progress bar and its subcomponents''' ... @@ -378,7 +381,7 @@ class FormatCustomText(FormatWidgetMixin, WidgetBase): def __init__(self, format, mapping=..., **kwargs) -> None: ... - def update_mapping(self, **mapping): # -> None: + def update_mapping(self, **mapping): ... def __call__(self, progress, data): @@ -410,10 +413,10 @@ class MultiRangeBar(Bar, VariableMixin): def __init__(self, name, markers, **kwargs) -> None: ... - def get_values(self, progress, data): # -> list[Any]: + def get_values(self, progress, data): ... - def __call__(self, progress, data, width): # -> str: + def __call__(self, progress, data, width): '''Updates the progress bar and its subcomponents''' ... @@ -423,7 +426,7 @@ class MultiProgressBar(MultiRangeBar): def __init__(self, name, markers=..., **kwargs) -> None: ... - def get_values(self, progress, data): # -> list[int]: + def get_values(self, progress, data): ... @@ -463,7 +466,7 @@ class GranularBar(AutoWidthWidgetBase): ''' ... - def __call__(self, progress, data, width): # -> str: + def __call__(self, progress, data, width): ... @@ -491,7 +494,7 @@ class Variable(FormatWidgetMixin, VariableMixin, WidgetBase): '''Creates a Variable associated with the given name.''' ... - def __call__(self, progress, data): # -> str: + def __call__(self, progress, data): ... @@ -510,10 +513,10 @@ class CurrentTime(FormatWidgetMixin, TimeSensitiveWidgetBase): def __call__(self, progress, data): ... - def current_datetime(self): # -> datetime: + def current_datetime(self): ... - def current_time(self): # -> _Time: + def current_time(self): ...