Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
* `PLAYWRIGHT_CONTEXTS` (type `dict[str, dict]`, default `{}`)

A dictionary which defines Browser contexts to be created on startup.
It should be a mapping of (name, keyword arguments) For instance:
It should be a mapping of (name, keyword arguments). For instance:
```python
{
"first": {
Expand All @@ -87,8 +87,8 @@ TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
},
}
```
If no contexts are defined, a default context (called `default`) is created.
The arguments passed here take precedence over the ones defined in `PLAYWRIGHT_CONTEXT_ARGS`.
A default context (called `default`) is created if no contexts are defined,
this will be used by all requests which do not explicitly specify a context.
See the docs for [`Browser.new_context`](https://playwright.dev/python/docs/api/class-browser#browser-new-context).

* `PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT` (type `Optional[float]`, default `None`)
Expand Down Expand Up @@ -518,15 +518,6 @@ for more information about deprecations and removals.

### Currently deprecated features

* `PLAYWRIGHT_CONTEXT_ARGS` setting (type `dict`, default `{}`)

A dictionary with default keyword arguments to be
used when creating the "default" Browser context.

Deprecated since
[`v0.0.4`](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.4),
use the `PLAYWRIGHT_CONTEXTS` setting instead

* `scrapy_playwright.page.PageCoroutine` class

Deprecated since
Expand Down
5 changes: 5 additions & 0 deletions changelog.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# scrapy-playwright changelog

### v0.0.15 (2022-NN-NN)

* Remove deprecated `PLAYWRIGHT_CONTEXT_ARGS` setting


### [v0.0.14](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.14) (2022-03-26)

* Renamed `PageCoroutine` to `PageMethod` (`PageCoroutine` is now deprecated)
Expand Down
19 changes: 1 addition & 18 deletions scrapy_playwright/handler.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import asyncio
import logging
import warnings
from collections import defaultdict
from contextlib import suppress
from inspect import isawaitable
from ipaddress import ip_address
Expand Down Expand Up @@ -90,23 +89,7 @@ def __init__(self, crawler: Crawler) -> None:
else:
self.process_request_headers = use_scrapy_headers

default_context_kwargs: dict = {}
if "PLAYWRIGHT_CONTEXT_ARGS" in crawler.settings:
default_context_kwargs = crawler.settings.getdict("PLAYWRIGHT_CONTEXT_ARGS")
warnings.warn(
"The PLAYWRIGHT_CONTEXT_ARGS setting is deprecated, please use"
" PLAYWRIGHT_CONTEXTS instead. Keyword arguments defined in"
" PLAYWRIGHT_CONTEXT_ARGS will be used when creating the 'default' context",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.context_kwargs: defaultdict = defaultdict(dict)
for name, kwargs in (crawler.settings.getdict("PLAYWRIGHT_CONTEXTS") or {}).items():
if name == "default":
self.context_kwargs[name] = default_context_kwargs
self.context_kwargs[name].update(kwargs)
if "default" not in self.context_kwargs and default_context_kwargs:
self.context_kwargs["default"] = default_context_kwargs
self.context_kwargs: dict = crawler.settings.getdict("PLAYWRIGHT_CONTEXTS")
self.contexts: Dict[str, BrowserContext] = {}
self.context_semaphores: Dict[str, asyncio.Semaphore] = {}

Expand Down
45 changes: 0 additions & 45 deletions tests/test_browser_contexts.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import asyncio
import platform
import warnings

import pytest
from scrapy import Spider, Request
from scrapy.exceptions import ScrapyDeprecationWarning

from tests import make_handler
from tests.mockserver import StaticMockServer
Expand Down Expand Up @@ -136,49 +134,6 @@ async def test_contexts_dynamic(self):
assert cookie["value"] == "qwerty"
assert cookie["domain"] == "example.org"

@pytest.mark.asyncio
async def test_deprecated_setting(self):
settings = {
"PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
"PLAYWRIGHT_CONTEXT_ARGS": {
"storage_state": {
"cookies": [
{
"url": "https://example.org",
"name": "asdf",
"value": "qwerty",
},
],
},
},
}
with warnings.catch_warnings(record=True) as warning_list:
async with make_handler(settings) as handler:
assert warning_list[0].category is ScrapyDeprecationWarning
assert str(warning_list[0].message) == (
"The PLAYWRIGHT_CONTEXT_ARGS setting is deprecated, please use"
" PLAYWRIGHT_CONTEXTS instead. Keyword arguments defined in"
" PLAYWRIGHT_CONTEXT_ARGS will be used when creating the 'default' context"
)
assert len(handler.contexts) == 1
assert len(handler.context_semaphores) == 1

with StaticMockServer() as server:
meta = {
"playwright": True,
"playwright_include_page": True,
}
req = Request(server.urljoin("/index.html"), meta=meta)
resp = await handler._download_request(req, Spider("foo"))

page = resp.meta["playwright_page"]
storage_state = await page.context.storage_state()
await page.close()
cookie = storage_state["cookies"][0]
assert cookie["name"] == "asdf"
assert cookie["value"] == "qwerty"
assert cookie["domain"] == "example.org"


class TestCaseMultipleContextsChromium(MixinTestCaseMultipleContexts):
browser_type = "chromium"
Expand Down