Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
557d340
using local html instead of relying on external server in testing
bmos Dec 2, 2025
cd20aa5
fix firefox browser remaining open after testing
bmos Dec 2, 2025
b9a50d5
don't rely on google cookies for cookie testing
bmos Dec 2, 2025
d149121
add driver to session cookie transfer test
bmos Dec 2, 2025
fa7d12d
better solution to fix browsers remaining open after testing
bmos Dec 2, 2025
f5405a7
update codeql workflow to fix failing check
bmos Dec 2, 2025
bbd8575
add test for creating session object headless
bmos Dec 2, 2025
705412f
add test for creating session object with webdriver options
bmos Dec 2, 2025
e26e797
add test for domain filter for session to driver cookie transfer
bmos Dec 2, 2025
7ece850
add test for creating session without explicit driver selection
bmos Dec 2, 2025
5664cb5
retry creation of firefox driver to work around timeouts when run in …
bmos Dec 2, 2025
ec014bf
add more session initialization tests
bmos Dec 2, 2025
73f0ce0
update naming of test_session tests
bmos Dec 2, 2025
68cdd41
create test_mixin
bmos Dec 2, 2025
59f95d2
fix type hint in requestium_mixin.py
bmos Dec 2, 2025
566b732
add parameterized test_ensure_element
bmos Dec 2, 2025
670bf57
Revert "retry creation of firefox driver to work around timeouts when…
bmos Dec 2, 2025
51625fb
skip non-headless firefox testing in CI workflows
bmos Dec 2, 2025
24ce87e
run chrome tests with CI compatibility tweaks
bmos Dec 2, 2025
0a3ad83
wait for page load before continuing in test_simple_page_load
bmos Dec 2, 2025
005af3f
reusable session fixture across tests for speed and consistency
bmos Dec 2, 2025
63893ae
more readable session fixture
bmos Dec 2, 2025
fcfc2aa
add missing type to session fixture
bmos Dec 3, 2025
c273d97
reset user agent state in user agent tests
bmos Dec 3, 2025
af40acc
reset cookie state in cookie test fixture
bmos Dec 3, 2025
43c2169
add test_ensure_click
bmos Dec 3, 2025
66528ab
add test_ensure_add_cookie and test_ensure_add_cookie_domain_override
bmos Dec 3, 2025
875e27b
simplify cookie tests with assert_first_cookie_matches utility function
bmos Dec 3, 2025
629eebb
add assert function to check webelement text
bmos Dec 3, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
- python

steps:
- uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a
- uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2
with:
disable-sudo: true
egress-policy: block
Expand All @@ -36,15 +36,16 @@ jobs:
github.com:443
objects.githubusercontent.com:443
uploads.github.com:443
release-assets.githubusercontent.com

- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3

- uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee
- uses: github/codeql-action/init@e12f0178983d466f2f6028f5cc7a6d786fd97f4b
with:
languages: ${{ matrix.language }}

- uses: github/codeql-action/autobuild@0499de31b99561a6d14a36a5f662c2a54f91beee
- uses: github/codeql-action/autobuild@e12f0178983d466f2f6028f5cc7a6d786fd97f4b

- uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee
- uses: github/codeql-action/analyze@e12f0178983d466f2f6028f5cc7a6d786fd97f4b
with:
category: /language:${{matrix.language}}
4 changes: 2 additions & 2 deletions requestium/requestium_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import tldextract
from parsel.selector import Selector, SelectorList
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.by import By, ByType
from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait
Expand Down Expand Up @@ -130,7 +130,7 @@ def ensure_element_by_class_name(self, selector: str, state: str | None = "prese
def ensure_element_by_css_selector(self, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
return self.ensure_element(By.CSS_SELECTOR, selector, state, timeout)

def ensure_element(self, locator: str, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
def ensure_element(self, locator: ByType | str, selector: str, state: str | None = "present", timeout: float | None = None) -> WebElement | None:
"""
Wait until an element appears or disappears in the browser.

Expand Down
91 changes: 60 additions & 31 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,49 +1,78 @@
# import os
# import shutil
import contextlib
from collections.abc import Generator
from typing import TYPE_CHECKING, cast

import pytest
from _pytest.fixtures import FixtureRequest
from selenium import webdriver
from selenium.common import WebDriverException

import requestium

if TYPE_CHECKING:
from requestium.requestium_mixin import DriverMixin

# ruff: noqa FBT003


@pytest.fixture(scope="module")
def example_html() -> str:
return """
<html>
<head><title>The Internet</title></head>
<body>
<h1>Test Header 1</h1>
<h2 id="test-header">Test Header 2</h2>
<h3 class="test-header-3">Test Header 3</h3>
<p class="body-text">Test Paragraph 1</p>
<button>Click Me</button>
<p><a href="example.com" name="link-paragraph">Test Link 1</a></p>
<p><a href="example.com">Test Link 2</a></p>
</body>
</html>
"""


def _create_chrome_driver(headless: bool) -> webdriver.Chrome:
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
if headless:
options.add_argument("--headless=new")
return webdriver.Chrome(options=options)


def _create_firefox_driver(headless: bool) -> webdriver.Firefox:
options = webdriver.FirefoxOptions()
options.set_preference("browser.cache.disk.enable", False)
options.set_preference("browser.cache.memory.enable", False)
options.set_preference("browser.cache.offline.enable", False)
options.set_preference("network.http.use-cache", False)
if headless:
options.add_argument("--headless")
return webdriver.Firefox(options=options)


@pytest.fixture(
params=[
"chrome-headless",
"chrome",
# "chrome-custom-path",
"firefox-headless",
"firefox",
]
params=["chrome-headless", "chrome", "firefox-headless", "firefox"],
scope="module",
)
def session(request): # noqa: ANN001, ANN201
def session(request: FixtureRequest) -> Generator[requestium.Session, None, None]:
driver_type = request.param
browser, _, mode = driver_type.partition("-")
headless = mode == "headless"

if driver_type == "chrome-headless":
options = webdriver.ChromeOptions()
options.add_argument("--headless=new")
driver = webdriver.Chrome(options=options)
elif driver_type == "chrome":
driver = webdriver.Chrome()
# elif driver_type == "chrome-custom-path":
# chromedriver_name = "chromedriver"
# custom_path = shutil.which(chromedriver_name)
# assert custom_path, f"'{chromedriver_name}' not found in PATH."
# assert os.path.exists(custom_path), f"Custom chromedriver not found at {custom_path}."
# driver = webdriver.Chrome(service=webdriver.ChromeService(executable_path=custom_path))
elif driver_type == "firefox-headless":
options = webdriver.FirefoxOptions()
options.add_argument("--headless")
driver = webdriver.Firefox(options=options)
elif driver_type == "firefox":
driver = webdriver.Firefox()
driver: webdriver.Chrome | webdriver.Firefox
if browser == "chrome":
driver = _create_chrome_driver(headless)
elif browser == "firefox":
driver = _create_firefox_driver(headless)
else:
msg = f"Unknown driver type: {driver_type}"
msg = f"Unknown driver type: {browser}"
raise ValueError(msg)

with requestium.Session(driver=cast("DriverMixin", driver)) as session:
yield session
session = requestium.Session(driver=cast("DriverMixin", driver))
yield session

with contextlib.suppress(WebDriverException, OSError):
driver.quit()
Binary file added tests/resources/test_extension.crx
Binary file not shown.
25 changes: 0 additions & 25 deletions tests/test__start_chrome_driver.py

This file was deleted.

95 changes: 84 additions & 11 deletions tests/test_cookies.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,95 @@
import pytest
from _pytest.fixtures import FixtureRequest
from selenium.common import InvalidCookieDomainException

import requestium.requestium


def test_transfer_session_cookies_to_driver(session: requestium.Session) -> None:
assert session.cookies.keys() == []
response = session.get("http://google.com/")
assert response.cookies.keys().sort() == ["AEC", "NID"].sort()
session.transfer_session_cookies_to_driver()
assert session.cookies.keys() == ["AEC", "NID"]
@pytest.fixture(
params=[
{"name": "session_id", "value": "abc123", "domain": "example.com", "path": "/"},
{"name": "user_token", "value": "xyz789", "domain": "example.com", "path": "/"},
],
ids=["session_id", "user_token"],
scope="module",
)
def cookie_data(request: FixtureRequest) -> dict[str, str]:
return request.param


@pytest.fixture
def clean_session(session: requestium.Session) -> requestium.Session:
"""Ensure cookies are cleared before each test."""
session.cookies.clear()
session.driver.delete_all_cookies()
return session


def assert_first_cookie_matches(driver_cookies: list[dict], expected: dict[str, str]) -> None:
"""Verify the first cookie in a list matches expected values."""
assert len(driver_cookies) == 1

cookie = driver_cookies[0]
assert cookie["name"] == expected["name"]
assert cookie["value"] == expected["value"]
assert cookie["domain"] in {expected["domain"], f".{expected['domain']}"}
assert cookie["path"] == expected["path"]


def test_ensure_add_cookie(clean_session: requestium.Session, cookie_data: dict[str, str]) -> None:
clean_session.driver.get("https://google.com")
clean_session.driver.delete_all_cookies()
clean_session.driver.ensure_add_cookie(cookie_data)

assert_first_cookie_matches(clean_session.driver.get_cookies(), cookie_data)


def test_ensure_add_cookie_domain_override(clean_session: requestium.Session, cookie_data: dict[str, str]) -> None:
override_domain = "example.net"

clean_session.driver.get("https://google.com")
clean_session.driver.delete_all_cookies()
clean_session.driver.ensure_add_cookie(cookie_data, override_domain=override_domain)

expected = {**cookie_data, "domain": override_domain}
assert_first_cookie_matches(clean_session.driver.get_cookies(), expected)


def test_transfer_driver_cookies_to_session(clean_session: requestium.Session, cookie_data: dict[str, str]) -> None:
clean_session.driver.get(f"https://{cookie_data['domain']}")
clean_session.driver.add_cookie(cookie_data)

assert not clean_session.cookies.keys()
clean_session.transfer_driver_cookies_to_session()
assert clean_session.cookies.keys() == [cookie_data["name"]]


def test_transfer_session_cookies_to_driver(clean_session: requestium.Session, cookie_data: dict[str, str]) -> None:
clean_session.get(f"http://{cookie_data['domain']}")
clean_session.cookies.set(name=cookie_data["name"], value=cookie_data["value"], domain=cookie_data["domain"], path=cookie_data["path"])

assert not clean_session.driver.get_cookies()
clean_session.transfer_session_cookies_to_driver()
assert_first_cookie_matches(clean_session.driver.get_cookies(), cookie_data)


def test_transfer_session_cookies_to_driver_domain_filter(clean_session: requestium.Session, cookie_data: dict[str, str]) -> None:
clean_session.get(f"http://{cookie_data['domain']}")
clean_session.cookies.set(name="junk_cookie", value="sfkjn782", domain="google.com", path=cookie_data["path"])
clean_session.cookies.set(name=cookie_data["name"], value=cookie_data["value"], domain=cookie_data["domain"], path=cookie_data["path"])

assert not clean_session.driver.get_cookies()
clean_session.transfer_session_cookies_to_driver(domain=cookie_data["domain"])
assert_first_cookie_matches(clean_session.driver.get_cookies(), cookie_data)


def test_transfer_session_cookies_to_driver_no_domain_error(session: requestium.Session) -> None:
with (
pytest.raises(
InvalidCookieDomainException,
match="Trying to transfer cookies to selenium without specifying a domain and without having visited any page in the current session",
),
session.cookies.clear()
session.driver.delete_all_cookies()
session._last_requests_url = None

with pytest.raises(
InvalidCookieDomainException,
match="Trying to transfer cookies to selenium without specifying a domain and without having visited any page in the current session",
):
session.transfer_session_cookies_to_driver()
9 changes: 0 additions & 9 deletions tests/test_ensure_elements_deprecation.py

This file was deleted.

Loading
Loading