Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,6 @@ jobs:

runs-on: ubuntu-latest

permissions:
pull-requests: write

steps:
- uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf
with:
Expand Down Expand Up @@ -114,7 +111,6 @@ jobs:
runs-on: ubuntu-latest

permissions:
pull-requests: write
security-events: write

steps:
Expand Down Expand Up @@ -159,6 +155,43 @@ jobs:
run: |
ruff check --output-format=github .

mypy:
name: Mypy type checking

runs-on: ubuntu-latest

steps:
- uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf
with:
disable-sudo: true
egress-policy: block
allowed-endpoints: >
api.github.com:443
files.pythonhosted.org:443
github.com:443
pypi.org:443

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

- uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55
with:
python-version: '3.13'
cache: pip

- uses: install-pinned/uv@5e770af195bb60f7bafe5430e7c5045bc2894b2a

- run: uv pip install --system -e .[dev]

- id: cache-mypy
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
with:
path: .mypy_cache
key: ${{ runner.os }}-mypy-3.13-${{ hashFiles('pyproject.toml') }}

- id: run-mypy
run: |
mypy .

bandit:
name: Bandit security

Expand Down Expand Up @@ -190,7 +223,7 @@ jobs:
- run: uv pip install --system -e .[dev]

- id: run-bandit-sarif
run: >
run: |
bandit --confidence-level 'medium' --format 'sarif' --output 'results.sarif' --recursive 'requestium'

- uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47
Expand All @@ -200,7 +233,7 @@ jobs:

- id: run-bandit
if: failure() && contains('["failure"]', steps.run-bandit-sarif.outcome)
run: >
run: |
bandit --confidence-level 'medium' --recursive 'requestium'

coverage:
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,13 @@ issues = "https://github.com/tryolabs/requestium/issues"
dev = [
"bandit[sarif]==1.8.3",
"coverage==7.8.0",
"mypy==1.15.0",
"pre-commit==4.2.0",
"pytest-cov==6.1.1",
"pytest-xdist==3.6.1",
"pytest==8.3.5",
"ruff==0.11.5",
"types-requests==2.32.0.20250328",
]

[tool.ruff]
Expand Down
8 changes: 5 additions & 3 deletions requestium/requestium.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from .requestium_session import Session # noqa: F401
from .requestium_mixin import ( # noqa: F401
DriverMixin, # noqa: F401
_ensure_click, # noqa: F401
) # noqa: F401
from .requestium_response import RequestiumResponse # noqa: F401
from .requestium_mixin import DriverMixin # noqa: F401
from .requestium_mixin import _ensure_click # noqa: F401
from .requestium_session import Session # noqa: F401
7 changes: 4 additions & 3 deletions requestium/requestium_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@
from parsel.selector import Selector, SelectorList
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait


class DriverMixin:
class DriverMixin(RemoteWebDriver):
"""Provides helper methods to our driver classes"""

def __init__(self, *args, **kwargs) -> None:
Expand All @@ -26,7 +27,7 @@ def try_add_cookie(self, cookie) -> bool:
try:
self.add_cookie(cookie)
except WebDriverException as e:
if not e.msg.__contains__("Couldn't add the following cookie to the webdriver"):
if e.msg and not e.msg.__contains__("Couldn't add the following cookie to the webdriver"):
raise WebDriverException from e
pass
return self.is_cookie_in_driver(cookie)
Expand Down Expand Up @@ -175,7 +176,7 @@ def ensure_element(self, locator: str, selector: str, state: str = "present", ti
# sometimes needs some time before it can click an item, specially if it needs to
# scroll into it first. This method ensures clicks don't fail because of this.
if element:
element.ensure_click = functools.partial(_ensure_click, element)
element.ensure_click = functools.partial(_ensure_click, element) # type: ignore[attr-defined]
return element

@property
Expand Down
9 changes: 5 additions & 4 deletions requestium/requestium_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
from typing import Any, Optional

import requests
import tldextract
from selenium import webdriver
from selenium.common import InvalidCookieDomainException
import tldextract
from selenium.webdriver import ChromeService

from .requestium_mixin import DriverMixin
Expand All @@ -31,7 +31,7 @@ def __init__(
headless: Optional[bool] = None,
default_timeout: float = 5,
webdriver_options: Optional[dict[str, Any]] = None,
driver=None,
driver: Optional[DriverMixin] = None,
) -> None:
super().__init__()

Expand All @@ -42,7 +42,7 @@ def __init__(
self.default_timeout = default_timeout
self.webdriver_options = webdriver_options
self._driver = driver
self._last_requests_url = None
self._last_requests_url: Optional[str] = None

if not self._driver:
self._driver_initializer = functools.partial(self._start_chrome_browser, headless=headless)
Expand Down Expand Up @@ -113,7 +113,8 @@ def transfer_session_cookies_to_driver(self, domain: Optional[str] = None) -> No
"""
if not domain and self._last_requests_url:
domain = tldextract.extract(self._last_requests_url).registered_domain
elif not domain and not self._last_requests_url:

if not domain:
raise InvalidCookieDomainException(
"Trying to transfer cookies to selenium without specifying a domain and without having visited any page in the current session"
)
Expand Down
Loading