Merge branch 'master' into python3_str

.github/workflows/python-release.yml

@@ -8,11 +8,10 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
       with:
-        python-version: 3.9
+        python-version: 3.10
     - name: Install build requirements
       run: python -m pip install wheel
     - name: Build package

.github/workflows/python-test.yml

@@ -12,11 +12,10 @@ jobs:
           - format
           - mypy
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
       with:
-        python-version: 3.9
+        python-version: "3.10"
     - name: Install dependencies
       run: pip install tox
     - name: Validate formatting
@@ -28,11 +27,11 @@ jobs:
       max-parallel: 4
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

lxml-stubs/etree.pyi

@@ -9,6 +9,7 @@ from typing import (
     Any,
     AnyStr,
     Callable,
+    Collection,
     Dict,
     Iterable,
     Iterator,
@@ -25,7 +26,7 @@ from typing import (
     overload,
 )
 
-from typing_extensions import Literal, Protocol, TypeGuard
+from typing_extensions import Literal, Protocol, TypeAlias, TypeGuard
 
 # dummy for missing stubs
 def __getattr__(name: str) -> Any: ...
@@ -44,7 +45,7 @@ _AnySmartStr = Union[
 ]
 _TagName = Union[str, bytes, QName]
 # _TagSelector also allows Element, Comment, ProcessingInstruction
-_TagSelector = Union[str, bytes, QName, Any]
+_TagSelector = Union[_TagName, Collection[_TagSelector], Any]
 # XPath object - http://lxml.de/xpathxslt.html#xpath-return-values
 _XPathObject = Union[
     bool,
@@ -183,8 +184,8 @@ class _Element(Iterable["_Element"], Sized):
     def iterchildren(
         self,
         tag: Optional[_TagSelector] = ...,
-        reversed: bool = False,
         *tags: _TagSelector,
+        reversed: bool = False,
     ) -> Iterator[_Element]: ...
     iterdescendants = iter
     def iterfind(
@@ -193,14 +194,14 @@ class _Element(Iterable["_Element"], Sized):
     def itersiblings(
         self,
         tag: Optional[_TagSelector] = ...,
-        preceding: bool = False,
         *tags: _TagSelector,
+        preceding: bool = False,
     ) -> Iterator[_Element]: ...
     def itertext(
         self,
         tag: Optional[_TagSelector] = ...,
-        with_tail: bool = False,
         *tags: _TagSelector,
+        with_tail: bool = False,
     ) -> Iterator[_StrResult]: ...
     def keys(self) -> Sequence[_StrResult]: ...
     def makeelement(
@@ -715,3 +716,64 @@ class TreeBuilder:
     def start(self, tag: _TagName, attrib: Dict[_StrOrBytes, _StrOrBytes]) -> None: ...
 
 def iselement(element: Any) -> TypeGuard[_Element]: ...
+
+_ParseEventType: TypeAlias = Literal[
+    "start", "end", "start-ns", "end-ns", "comment", "pi"
+]
+_ParseEvent: TypeAlias = Union[
+    tuple[Literal["start"], _Element],
+    tuple[Literal["end"], _Element],
+    tuple[Literal["start-ns"], Tuple[_AnyStr, _AnyStr]],
+    tuple[Literal["end-ns"], None],
+    tuple[Literal["comment"], _Comment],
+    tuple[Literal["pi"], _ProcessingInstruction],
+]
+
+class XMLPullParser(XMLParser):
+    def __init__(
+        self,
+        events: Optional[Iterable[_ParseEventType]] = ...,
+        *,
+        tag: Optional[_TagSelector] = ...,
+        base_url: Optional[_AnyStr] = ...,
+        encoding: Optional[_AnyStr] = ...,
+        attribute_defaults: bool = ...,
+        dtd_validation: bool = ...,
+        load_dtd: bool = ...,
+        no_network: bool = ...,
+        ns_clean: bool = ...,
+        recover: bool = ...,
+        schema: Optional[XMLSchema] = ...,
+        huge_tree: bool = ...,
+        remove_blank_text: bool = ...,
+        resolve_entities: bool = ...,
+        remove_comments: bool = ...,
+        remove_pis: bool = ...,
+        strip_cdata: bool = ...,
+        collect_ids: bool = ...,
+        target: Optional[ParserTarget] = ...,
+        compact: bool = ...,
+    ) -> None: ...
+    def read_events(self) -> Iterator[_ParseEvent]: ...
+
+class HTMLPullParser(HTMLParser):
+    def __init__(
+        self,
+        events: Optional[Iterable[_ParseEventType]] = ...,
+        *,
+        tag: Optional[_TagSelector] = ...,
+        base_url: Optional[_AnyStr] = ...,
+        encoding: Optional[_AnyStr] = ...,
+        collect_ids: bool = ...,
+        compact: bool = ...,
+        huge_tree: bool = ...,
+        no_network: bool = ...,
+        recover: bool = ...,
+        remove_blank_text: bool = ...,
+        remove_comments: bool = ...,
+        remove_pis: bool = ...,
+        schema: Optional[XMLSchema] = ...,
+        strip_cdata: bool = ...,
+        target: Optional[ParserTarget] = ...,
+    ) -> None: ...
+    def read_events(self) -> Iterator[_ParseEvent]: ...

setup.py

@@ -3,9 +3,10 @@
 from setuptools import setup
 
 tests_require = [
-    "coverage[toml]==5.2",
-    "pytest>=6.0.0",
-    "pytest-mypy-plugins==1.9.3"
+    "coverage[toml]>=7.2.5",
+    "pytest>=7.3.0",
+    "pytest-mypy-plugins>=1.10.1",
+    "mypy>=1.2.0"
 ]
 
 with open("README.md") as fh:

test-data/test-etree.yml

@@ -34,7 +34,7 @@
     main: |
         from lxml import etree
         element = etree.Element("foo")
-        reveal_type(element.iterchildren)  # N: Revealed type is "def (tag: Union[builtins.str, builtins.bytes, lxml.etree.QName, Any, None] =, reversed: builtins.bool =, *tags: Union[builtins.str, builtins.bytes, lxml.etree.QName, Any]) -> typing.Iterator[lxml.etree._Element]"
+        reveal_type(element.iterchildren)  # N: Revealed type is "def (tag: Union[Union[builtins.str, builtins.bytes, lxml.etree.QName], typing.Collection[Union[builtins.str, builtins.bytes, lxml.etree.QName, typing.Collection[...], ...]], Any, None] =, *tags: Union[builtins.str, builtins.bytes, lxml.etree.QName, typing.Collection[...], ...], *, reversed: builtins.bool =) -> typing.Iterator[lxml.etree._Element]"
         result = element.iterchildren("my-attr")
         reveal_type(result)  # N: Revealed type is "typing.Iterator[lxml.etree._Element]"
 
@@ -101,3 +101,39 @@
         from lxml import etree
         document = etree.XML("<doc></doc>", parser=etree.XMLParser(), base_url="http://example.com/")
         reveal_type(document)  # N: Revealed type is "lxml.etree._Element"
+-   case: etree_XMLPullParser
+    disable_cache: true
+    main: |
+        from lxml import etree
+        parser = etree.XMLPullParser()
+        event = next(parser.read_events())
+        if event[0] == "start":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Element"
+        elif event[0] == "end":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Element"
+        elif event[0] == "start-ns":
+            reveal_type(event[1])  # N: Revealed type is "Tuple[Union[builtins.str, builtins.bytes], Union[builtins.str, builtins.bytes]]"
+        elif event[0] == "end-ns":
+            reveal_type(event[1])  # N: Revealed type is "None"
+        elif event[0] == "comment":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Comment"
+        elif event[0] == "pi":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._ProcessingInstruction"
+-   case: etree_HTMLPullParser
+    disable_cache: true
+    main: |
+        from lxml import etree
+        parser = etree.HTMLPullParser()
+        event = next(parser.read_events())
+        if event[0] == "start":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Element"
+        elif event[0] == "end":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Element"
+        elif event[0] == "start-ns":
+            reveal_type(event[1])  # N: Revealed type is "Tuple[Union[builtins.str, builtins.bytes], Union[builtins.str, builtins.bytes]]"
+        elif event[0] == "end-ns":
+            reveal_type(event[1])  # N: Revealed type is "None"
+        elif event[0] == "comment":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._Comment"
+        elif event[0] == "pi":
+            reveal_type(event[1])  # N: Revealed type is "lxml.etree._ProcessingInstruction"

tox.ini

@@ -1,12 +1,12 @@
 [tox]
-envlist = format,mypy,py36,py37,py38,py39,py310
+envlist = format,mypy,py37,py38,py39,py310,py311
 
 [testenv]
 commands = pytest {posargs}
 extras = test
 
 [testenv:format]
-basepython = python3.9
+basepython = python3.10
 deps =
     black
     isort>=5
@@ -16,9 +16,9 @@ commands =
     black --check --diff -v lxml-stubs/
 
 [testenv:mypy]
-basepython = python3.9
+basepython = python3.10
 deps =
-    mypy
+    mypy>=1.2.0
 skip_install = true
 commands =
     mypy lxml-stubs