feat: update the scrapers (#70)

- add globs to input
- update input schema
- update READMEs
- update packages

Author: AndreyBykov

package-lock.json

@@ -55,7 +55,7 @@
         "@types/content-type": "^1.1.5",
         "@types/fs-extra": "^9.0.13",
         "@types/jest": "^28.1.5",
-        "@types/node": "^18.0.3",
+        "@types/node": "^18.7.18",
         "@types/rimraf": "^3.0.2",
         "@types/semver": "^7.3.10",
         "@types/tough-cookie": "^4.0.2",
@@ -66,7 +66,7 @@
         "crawlee": "^3.0.0",
         "playwright": "^1.25.0",
         "puppeteer": "~17.0.0",
-        "eslint": "^8.19.0",
+        "eslint": "~8.22.0",
         "fs-extra": "^10.1.0",
         "gen-esm-wrapper": "^1.1.3",
         "husky": "^8.0.1",
@@ -77,7 +77,7 @@
         "ts-jest": "^28.0.5",
         "ts-node": "^10.8.2",
         "turbo": "1.4.6",
-        "typescript": "~4.8.2"
+        "typescript": "~4.8.3"
     },
     "packageManager": "npm@8.19.1"
 }

package.json

@@ -93,7 +93,7 @@ This is useful for determining which start URL is currently loaded, in order to
 
 ### Link selector
 
-The **Link selector** (`linkSelector`) field contains a CSS selector that is used to find links to other web pages, i.e. `<a>` elements with the `href` attribute. On every page loaded, the scraper looks for all links matching the **Link selector**. It checks that the target URL matches one of the [**Pseudo-URLs**](#pseudo-urls)/[**Glob Patterns**](#glob-patterns), and if so then adds the URL to the request queue, to be loaded by the scraper later.
+The **Link selector** (`linkSelector`) field contains a CSS selector that is used to find links to other web pages, i.e. `<a>` elements with the `href` attribute. On every page loaded, the scraper looks for all links matching the **Link selector**. It checks that the target URL matches one of the [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls), and if so then adds the URL to the request queue, to be loaded by the scraper later.
 
 By default, new scrapers are created with the following selector that matches all links:
 
@@ -585,7 +585,8 @@ You might also want to see these other resources:
   A similar web scraping actor to Puppeteer Scraper, but using the [Playwright](https://github.com/microsoft/playwright) library instead.
 - [Actors documentation](https://docs.apify.com/actors) -
   Documentation for the Apify Actors cloud computing platform.
-- [Crawlee](https://crawlee.dev) - Learn how to build a new web scraper from scratch using the world's most popular web crawling and scraping library for Node.js.
+- [Apify SDK documentation](https://sdk.apify.com) - Learn more about the tools required to run your own Apify actors.
+- [Crawlee documentation](https://crawlee.dev) - Learn how to build a new web scraping project from scratch using the world's most popular web crawling and scraping library for Node.js.
 
 ## Upgrading
 

packages/actor-scraper/cheerio-scraper/README.md

@@ -11,7 +11,7 @@
     },
     "devDependencies": {
         "@apify/tsconfig": "^0.1.0",
-        "@types/node": "^18.7.16",
+        "@types/node": "^18.7.18",
         "ts-node": "^10.9.1",
         "typescript": "^4.8.3"
     },

packages/actor-scraper/cheerio-scraper/package.json

@@ -54,7 +54,7 @@ Optionally, each URL can be associated with custom user data - a JSON object tha
 
 The **Link selector** (`linkSelector`) field contains a CSS selector that is used to find links to other web pages (items with `href` attributes, e.g. `<div class="my-class" href="...">`).
 
-On every page loaded, the scraper looks for all links matching **Link selector**, and checks that the target URL matches one of the [**Pseudo-URLs**](#pseudo-urls)/[**Glob Patterns**](#glob-patterns). If it is a match, it then adds the URL to the request queue so that it's loaded by the scraper later on.
+On every page loaded, the scraper looks for all links matching **Link selector**, and checks that the target URL matches one of the [**Glob Patterns**](#glob-patterns)/[**Pseudo-URLs**](#pseudo-urls). If it is a match, it then adds the URL to the request queue so that it's loaded by the scraper later on.
 
 By default, new scrapers are created with the following selector that matches all links on any page:
 
@@ -106,8 +106,7 @@ will match the URL:
 http://www.example.com/search?do[load]=1
 ```
 
-Optionally, each pseudo-URL can be associated with user data that can be referenced from your **[Page function](#page-function)**
-using `context.request.label` to determine which kind of page is currently loaded in the browser.
+Optionally, each pseudo-URL can be associated with user data that can be referenced from your **[Page function](#page-function)** using `context.request.label` to determine which kind of page is currently loaded in the browser.
 
 Note that you don't need to use the **Pseudo-URLs** setting at all,
 because you can completely control which pages the scraper will access by calling `await context.enqueueRequest()`
@@ -134,12 +133,13 @@ const context = {
 
     // EXPOSED OBJECTS
     page, // Playwright.Page object.
-    request, // Request object.
+    request, // Crawlee.Request object.
     response, // Response object holding the status code and headers.
+    session, // Reference to the currently used session.
+    proxyInfo, // Object holding the url and other information about currently used Proxy.
     crawler, // Reference to the crawler object, with access to `browserPool`, `autoscaledPool`, and more.
     globalStore, // Represents an in memory store that can be used to share data across pageFunction invocations.
-    log, // Reference to log object.
-    playwrightUtils, // Reference to playwrightUtils namespace, containing various utilities for Playwright.
+    log, // Reference to Crawlee.utils.log.
     Actor, // Reference to the Actor class of Apify SDK.
     Apify, // Alias to the Actor class for back compatibility.
 
@@ -149,6 +149,11 @@ const context = {
     saveSnapshot, // Saves a screenshot and full HTML of the current page to the key value store.
     skipLinks, // Prevents enqueueing more links via glob patterns/Pseudo URLs on the current page.
     enqueueRequest, // Adds a page to the request queue.
+
+    // PLAYWRIGHT CONTEXT-AWARE UTILITY FUNCTIONS
+    injectJQuery, // Injects the jQuery library into a Playwright page.
+    sendRequest, // Sends request using got-scraping.
+    parseWithCheerio, // Returns Cheerio handle for page.content(), allowing to work with the data same way as with CheerioCrawler.
 };
 ```
 
@@ -186,8 +191,8 @@ This is a reference to the Playwright Page object, which enables you to use the
 
 #### **`request`**
 
-| Type   | Arguments | Returns                                                        |
-| ------ | --------- | -------------------------------------------------------------- |
+| Type   | Arguments | Returns                                                      |
+| ------ | --------- |--------------------------------------------------------------|
 | Object | -         | [Request](https://crawlee.dev/api/core/class/Request) object |
 
 An object with metadata about the currently crawled page, such as its URL, headers, and the number of retries.
@@ -219,6 +224,22 @@ See the [Request class](https://crawlee.dev/api/core/class/Request) for a previe
 
 The response object is produced by Playwright. Currently, we only pass the response's HTTP status code and headers to the `response` object.
 
+#### **`session`**
+
+| Type   | Arguments | Returns                                                       |
+| ------ | --------- |---------------------------------------------------------------|
+| Object | -         | [Session](https://crawlee.dev/api/core/class/Session) object  |
+
+Reference to the currently used session. See the [official documentation](https://crawlee.dev/api/core/class/Session) for more information.
+
+#### **`proxyInfo`**
+
+| Type   | Arguments | Returns                                                              |
+| ------ | --------- |----------------------------------------------------------------------|
+| Object | -         | [ProxyInfo](https://crawlee.dev/api/core/interface/ProxyInfo) object |
+
+Object holding the url and other information about currently used Proxy. See the [official documentation](https://crawlee.dev/api/core/interface/ProxyInfo) for more information.
+
 #### **`crawler`**
 
 | Type   | Arguments | Returns                                                                                        |
@@ -251,9 +272,9 @@ Refer to the [official documentation](https://crawlee.dev/api/playwright-crawler
 
 #### **`log`**
 
-| Type   | Arguments | Returns                                              |
-| ------ | --------- |------------------------------------------------------|
-| Object | -         | [log](https://crawlee.dev/api/core/class/Log) object |
+| Type   | Arguments | Returns                                                            |
+| ------ | --------- |--------------------------------------------------------------------|
+| Object | -         | [Crawlee.utils.log](https://crawlee.dev/api/core/class/Log) object |
 
 This should be used instead of JavaScript's built in `console.log` when logging in the Node.js context, as it automatically color-tags your logs, as well as allows the toggling of the visibility of log messages using options such as [Debug log](#debug-log) in [Advanced configuration](#advanced-configuration).
 
@@ -265,14 +286,6 @@ The most common `log` methods include:
 -   `context.log.error()`
 -   `context.log.exception()`
 
-#### **`playwrightUtils`**
-
-| Type   | Arguments | Returns                                                                                        |
-| ------ | --------- |------------------------------------------------------------------------------------------------|
-| Object | -         | [playwrightUtils](https://crawlee.dev/api/playwright-crawler/namespace/playwrightUtils) object |
-
-This is a namespace containing various utility functions for Playwright. Refer to the [official documentation](https://crawlee.dev/api/playwright-crawler/namespace/playwrightUtils) for more information.
-
 #### **`Actor`**
 
 | Type   | Arguments | Returns                                                           |
@@ -379,6 +392,55 @@ This method is a nice shorthand for
 await context.crawler.requestQueue.addRequest({ url: 'https://foo.bar/baz' })
 ```
 
+#### **`injectJQuery`**
+
+| Type     | Arguments | Returns          |
+| -------- |-----------|------------------|
+| Function | ()        | _Promise\<void>_ |
+
+> This function is async! Don't forget the `await` keyword!
+
+Injects the [jQuery](https://jquery.com/) library into a Playwright page. The injected jQuery will be set to the `window.$` variable, and will survive page navigations and reloads. Note that `injectJQuery()` does not affect the Playwright [`page.$()`](https://playwright.dev/docs/api/class-page#page-query-selector) function in any way.
+
+Usage:
+
+```JavaScript
+await context.injectJQuery();
+```
+
+#### **`sendRequest`**
+
+| Type     | Arguments                                    | Returns          |
+| -------- |----------------------------------------------|------------------|
+| Function | (overrideOptions?: Partial\<GotOptionsInit>) | _Promise\<void>_ |
+
+> This function is async! Don't forget the `await` keyword!
+
+This is a helper function that allows processing the context bound `Request` object through [`got-scraping`](https://github.com/apify/got-scraping). Some options, such as `url` or `method` could be overridden by providing `overrideOptions`. See the [official documentation](https://crawlee.dev/docs/guides/got-scraping#sendrequest-api) for full list of possible `overrideOptions` and more information.
+
+Usage:
+
+```JavaScript
+// Without overrideOptions
+await context.sendRequest();
+// With overrideOptions.url
+await context.sendRequest({ url: 'https://www.example.com' });
+```
+
+#### **`parseWithCheerio`**
+
+| Type     | Arguments | Returns                 |
+| -------- |-----------|-------------------------|
+| Function | ()        | _Promise\<CheerioRoot>_ |
+
+Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with CheerioCrawler.
+
+Usage:
+
+```JavaScript
+const $ = await context.parseWithCheerio();
+```
+
 ## Proxy Configuration
 
 The **Proxy configuration** (`proxyConfiguration`) option enables you to set proxies
@@ -421,7 +483,7 @@ The proxy configuration can be set programmatically when calling the actor using
 
 ## Browser Configuration
 
-### Browser Type
+### `launcher`
 
 The actor will use a Chromium browser by default. Alternatively, you can set it to use a Firefox browser instead.
 
@@ -516,7 +578,7 @@ The full object stored in the dataset would look as follows (in JSON format, inc
 }
 ```
 
-To download the results, call the [Get dataset items](https://apify.com/docs/api/v2#/reference/datasets/item-collection) API endpoint:
+To download the results, call the [Get dataset items](https://docs.apify.com/api/v2#/reference/datasets/item-collection) API endpoint:
 
 ```
 https://api.apify.com/v2/datasets/[DATASET_ID]/items?format=json
@@ -534,7 +596,8 @@ For more information, see [Datasets](https://apify.com/docs/storage#dataset) in
 That's it! You might also want to check out these other resources:
 
 -   [Actors documentation](https://apify.com/docs/actor) - Documentation for the Apify Actors cloud computing platform.
--   [Crawlee](https://crawlee.dev) - Learn how to build a new web scraping project from scratch using the world's most popular web crawling and scraping library for Node.js.
+-   [Apify SDK documentation](https://sdk.apify.com) - Learn more about the tools required to run your own Apify actors.
+-   [Crawlee documentation](https://crawlee.dev) - Learn how to build a new web scraping project from scratch using the world's most popular web crawling and scraping library for Node.js.
 -   [Cheerio Scraper](https://apify.com/apify/cheerio-scraper) - Another web scraping actor that downloads and processes pages in raw HTML for much higher performance.
 -   [Puppeteer Scraper](https://apify.com/apify/puppeteer-scraper) - A similar web scraping actor to Playwright Scraper, but using the [Puppeteer](https://github.com/puppeteer/puppeteer) library instead.
 -   [Web Scraper](https://apify.com/apify/web-scraper) - A similar web scraping actor to Playwright Scraper, but is simpler to use and only runs in the context of the browser. Uses the [Puppeteer](https://github.com/puppeteer/puppeteer) library.

packages/actor-scraper/playwright-scraper/README.md

@@ -14,7 +14,7 @@
     },
     "devDependencies": {
         "@apify/tsconfig": "^0.1.0",
-        "@types/node": "^18.7.16",
+        "@types/node": "^18.7.18",
         "ts-node": "^10.9.1",
         "typescript": "^4.8.3"
     },

packages/actor-scraper/playwright-scraper/package.json

@@ -12,7 +12,6 @@ import {
     PlaywrightCrawlerOptions,
     PlaywrightLaunchContext,
     BrowserCrawlerEnqueueLinksOptions,
-    playwrightUtils,
     log,
 } from '@crawlee/playwright';
 import { Awaitable, Dictionary } from '@crawlee/utils';
@@ -42,7 +41,6 @@ export class CrawlerSetup implements CrawlerSetupOptions {
     requestQueue: RequestQueue;
     keyValueStore: KeyValueStore;
     customData: unknown;
-    playwrightUtils = playwrightUtils;
     input: Input;
     maxSessionUsageCount: number;
     evaledPageFunction: (...args: unknown[]) => unknown;
@@ -216,15 +214,13 @@ export class CrawlerSetup implements CrawlerSetupOptions {
     }
 
     private _createNavigationHooks(options: PlaywrightCrawlerOptions) {
-        options.preNavigationHooks!.push(async ({ request, page, session }, gotoOptions) => {
+        options.preNavigationHooks!.push(async ({ request, page, session, blockRequests }, gotoOptions) => {
             // Attach a console listener to get all logs from Browser context.
             if (this.input.browserLog) browserTools.dumpConsole(page);
 
             // Prevent download of stylesheets and media, unless selected otherwise
             if (this.blockedUrlPatterns.length) {
-                await playwrightUtils.blockRequests(page, {
-                    urlPatterns: this.blockedUrlPatterns,
-                });
+                await blockRequests({ urlPatterns: this.blockedUrlPatterns });
             }
 
             // Add initial cookies, if any.
@@ -307,7 +303,6 @@ export class CrawlerSetup implements CrawlerSetupOptions {
                 requestQueue: this.requestQueue,
                 keyValueStore: this.keyValueStore,
                 customData: this.input.customData,
-                playwrightUtils: this.playwrightUtils,
             },
             pageFunctionArguments,
         };

packages/actor-scraper/playwright-scraper/src/internals/crawler_setup.ts

@@ -1,4 +1,4 @@
-FROM apify/actor-node-playwright:16 AS builder
+FROM apify/actor-node-puppeteer-chrome:16 AS builder
 
 COPY --chown=myuser package*.json ./
 

packages/actor-scraper/puppeteer-scraper/Dockerfile

@@ -9,24 +9,24 @@
             "title": "Start URLs",
             "type": "array",
             "description": "URLs to start with",
-            "prefill": [
-                {
-                    "url": "https://apify.com"
-                }
-            ],
+            "prefill": [{ "url": "https://crawlee.dev" }],
             "editor": "requestListSources"
         },
+        "globs": {
+            "title": "Glob Patterns",
+            "type": "array",
+            "description": "Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.",
+            "editor": "stringList",
+            "default": [],
+            "prefill": ["https://crawlee.dev/*/*"]
+        },
         "pseudoUrls": {
-            "title": "Pseudo-URLs",
+            "title": "Pseudo-URLs (deprecated)",
             "type": "array",
             "description": "Pseudo-URLs to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Pseudo-URLs will cause the scraper to enqueue all links matched by the Link selector.",
             "editor": "pseudoUrls",
             "default": [],
-            "prefill": [
-                {
-                    "purl": "https://apify.com[(/[\\w-]+)?]"
-                }
-            ]
+            "prefill": []
         },
         "linkSelector": {
             "title": "Link selector",