feat: add playwright scraper (#26)

Add playwright scraper

package.json

@@ -63,7 +63,9 @@
         "@typescript-eslint/eslint-plugin": "5.33.1",
         "@typescript-eslint/parser": "5.33.1",
         "commitlint": "^17.0.3",
-        "crawlee": "^3.0.3",
+        "crawlee": "^3.0.0",
+        "playwright": "^1.25.0",
+        "puppeteer": "^14.4.1",
         "eslint": "^8.19.0",
         "fs-extra": "^10.1.0",
         "gen-esm-wrapper": "^1.1.3",

packages/actor-scraper/cheerio-scraper/package.json

@@ -7,8 +7,8 @@
     "type": "module",
     "dependencies": {
         "@apify/scraper-tools": "^1.0.0",
-        "@crawlee/cheerio": "^3.0.3",
-        "apify": "^3.0.2"
+        "@crawlee/cheerio": "^3.0.0",
+        "apify": "^3.0.0"
     },
     "devDependencies": {
         "markdown-toc": "^1.2.0"

packages/actor-scraper/playwright-scraper/Dockerfile

@@ -0,0 +1,29 @@
+FROM apify/actor-node:16 AS builder
+
+COPY package*.json tsconfig.json crawlee?json ./
+COPY src ./src
+
+# Install all dependencies (including dev deps) and build the project
+RUN npm install --include=dev && npm run build
+
+# Create final image
+FROM apify/actor-node-playwright:16
+
+# Copy only necessary files (crawlee.json might not exist, so we need the regexp trick with `crawlee?json`)
+COPY --from=builder /usr/src/app/package.json /usr/src/app/crawlee?json ./
+COPY --from=builder /usr/src/app/dist ./dist
+COPY apify.json README.md INPUT_SCHEMA.json ./
+
+# Install only prod deps
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional \
+    && echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+ENV APIFY_DISABLE_OUTDATED_WARNING=1
+# Run compiled code
+CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod

packages/actor-scraper/playwright-scraper/INPUT_SCHEMA.json

@@ -0,0 +1,278 @@
+{
+    "title": "Playwright Scraper Input",
+    "description": "Use the following form to configure Playwright Scraper. The Start URLs and Page function are required and all other fields are optional. To learn more about the different options, click on their title or on the nearby question marks. For details about the Page function visit the actor's README in the ACTOR INFO tab.",
+    "type": "object",
+    "schemaVersion": 1,
+    "properties": {
+        "startUrls": {
+            "sectionCaption": "Basic configuration",
+            "title": "Start URLs",
+            "type": "array",
+            "description": "URLs to start with",
+            "prefill": [
+                {
+                    "url": "https://apify.com"
+                }
+            ],
+            "editor": "requestListSources"
+        },
+        "globs": {
+            "title": "Glob Patterns",
+            "type": "array",
+            "description": "Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.",
+            "editor": "stringList",
+            "default": [],
+            "prefill": [
+                "https://apify.com/**/*"
+            ]
+        },
+        "pseudoUrls": {
+            "title": "Pseudo-URLs (deprecated)",
+            "type": "array",
+            "description": "Pseudo-URLs to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Pseudo-URLs will cause the scraper to enqueue all links matched by the Link selector.",
+            "editor": "pseudoUrls",
+            "default": [],
+            "prefill": []
+        },
+        "linkSelector": {
+            "title": "Link selector",
+            "type": "string",
+            "description": "CSS selector matching elements with 'href' attributes that should be enqueued. To enqueue urls from <code><div class=\"my-class\" href=...></code> tags, you would enter <strong>div.my-class</strong>. Leave empty to ignore all links.",
+            "editor": "textfield",
+            "prefill": "a"
+        },
+        "keepUrlFragments": {
+            "title": "Keep URL fragments",
+            "type": "boolean",
+            "description": "URL fragments (the parts of URL after a <code>#</code>) are not considered when the scraper determines whether a URL has already been visited. This means that when adding URLs such as <code>https://example.com/#foo</code> and <code>https://example.com/#bar</code>, only the first will be visited. Turn this option on to tell the scraper to visit both.",
+            "default": false
+        },
+        "pageFunction": {
+            "title": "Page function",
+            "type": "string",
+            "description": "Function executed for each request",
+            "prefill": "async function pageFunction(context) {\n    const { page, request, log } = context;\n    const title = await page.title();\n    log.info(`URL: ${request.url} TITLE: ${title}`);\n    return {\n        url: request.url,\n        title\n    };\n}",
+            "editor": "javascript"
+        },
+        "proxyConfiguration": {
+            "sectionCaption": "Proxy configuration",
+            "title": "Proxy configuration",
+            "type": "object",
+            "description": "Choose to use no proxy, Apify Proxy, or provide custom proxy URLs.",
+            "prefill": { "useApifyProxy": true },
+            "default": { "useApifyProxy": false },
+            "editor": "proxy"
+        },
+        "proxyRotation": {
+            "title": "Proxy rotation",
+            "type": "string",
+            "description": "This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.",
+            "default": "RECOMMENDED",
+            "editor": "select",
+            "enum": [
+                "RECOMMENDED",
+                "PER_REQUEST",
+                "UNTIL_FAILURE"
+            ],
+            "enumTitles": [
+                "Use recommended settings",
+                "Rotate proxy after each request",
+                "Use one proxy until failure"
+            ]
+        },
+        "sessionPoolName": {
+            "title": "Session pool name",
+            "type": "string",
+            "description": "<b>Use only english alphanumeric characters dashes and underscores.</b> A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.",
+            "editor": "textfield",
+            "minLength": 3,
+            "maxLength": 200,
+            "pattern": "[0-9A-z-]"
+        },
+        "initialCookies": {
+            "title": "Initial cookies",
+            "type": "array",
+            "description": "The provided cookies will be pre-set to all pages the scraper opens.",
+            "default": [],
+            "prefill": [],
+            "editor": "json"
+        },
+        "launcher": {
+            "sectionCaption": "Browser configuration",
+            "title": "Browser Type",
+            "description": "Choose the browser to launch.",
+            "type": "string",
+            "editor": "select",
+            "enum": [
+                "chromium",
+                "firefox"
+            ],
+            "enumTitles": [
+                "Chromium",
+                "Firefox"
+            ],
+            "prefill": "chromium",
+            "default": "chromium"
+        },
+        "useChrome": {
+            "title": "Use Chrome (could only be used if Chromium Browser is selected)",
+            "type": "boolean",
+            "description": "The scraper will use a real Chrome browser instead of a Chromium masking as Chrome. Using this option may help with bypassing certain anti-scraping protections, but risks that the scraper will be unstable or not work at all.",
+            "default": false,
+            "groupCaption": "Browser masking",
+            "groupDescription": "Settings that help mask as a real user and prevent scraper detection."
+        },
+        "ignoreSslErrors": {
+            "title": "Ignore SSL errors",
+            "type": "boolean",
+            "description": "Scraper will ignore SSL certificate errors.",
+            "default": false,
+            "groupCaption": "Security",
+            "groupDescription": "Various options that help you disable browser security features such as HTTPS certificates and CORS."
+        },
+        "ignoreCorsAndCsp": {
+            "title": "Ignore CORS and CSP",
+            "type": "boolean",
+            "description": "Scraper will ignore CSP (content security policy) and CORS (cross origin resource sharing) settings of visited pages and requested domains. This enables you to freely use XHR/Fetch to make HTTP requests from the scraper.",
+            "default": false
+        },
+        "downloadMedia": {
+            "sectionCaption": "Performance and limits",
+            "title": "Download media",
+            "type": "boolean",
+            "description": "Scraper will download media such as images, fonts, videos and sounds. Disabling this may speed up the scrape, but certain websites could stop working correctly.",
+            "default": true,
+            "groupCaption": "Resources",
+            "groupDescription": "Settings that help to disable downloading web page resources."
+        },
+        "downloadCss": {
+            "title": "Download CSS",
+            "type": "boolean",
+            "description": "Scraper will download CSS stylesheets. Disabling this may speed up the scrape, but certain websites could stop working correctly.",
+            "default": true
+        },
+        "maxRequestRetries": {
+            "title": "Max request retries",
+            "type": "integer",
+            "description": "Maximum number of times the request for the page will be retried in case of an error. Setting it to 0 means that the request will be attempted once and will not be retried if it fails.",
+            "minimum": 0,
+            "default": 3,
+            "unit": "retries"
+        },
+        "maxPagesPerCrawl": {
+            "title": "Max pages per run",
+            "type": "integer",
+            "description": "Maximum number of pages that the scraper will open. 0 means unlimited.",
+            "minimum": 0,
+            "default": 0,
+            "unit": "pages"
+        },
+        "maxResultsPerCrawl": {
+            "title": "Max result records",
+            "type": "integer",
+            "description": "Maximum number of results that will be saved to dataset. The scraper will terminate afterwards. 0 means unlimited.",
+            "minimum": 0,
+            "default": 0,
+            "unit": "results"
+        },
+        "maxCrawlingDepth": {
+            "title": "Max crawling depth",
+            "type": "integer",
+            "description": "Defines how many links away from the StartURLs will the scraper descend. 0 means unlimited.",
+            "minimum": 0,
+            "default": 0
+        },
+        "maxConcurrency": {
+            "title": "Max concurrency",
+            "type": "integer",
+            "description": "Defines how many pages can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. Use this option to set a hard limit.",
+            "minimum": 1,
+            "default": 50
+        },
+        "pageLoadTimeoutSecs": {
+            "title": "Page load timeout",
+            "type": "integer",
+            "description": "Maximum time the scraper will allow a web page to load in seconds.",
+            "minimum": 1,
+            "default": 60,
+            "unit": "secs"
+        },
+        "pageFunctionTimeoutSecs": {
+            "title": "Page function timeout",
+            "type": "integer",
+            "description": "Maximum time the scraper will wait for the page function to execute in seconds.",
+            "minimum": 1,
+            "default": 60,
+            "unit": "secs"
+        },
+        "waitUntil": {
+            "title": "Navigation wait until",
+            "description": "The scraper will wait until the selected events are triggered in the page before executing the page function. Available events are <code>domcontentloaded</code>, <code>load</code> and <code>networkidle</code> <a href=\"https://playwright.dev/docs/api/class-page#page-goto-option-wait-until\" target=\"_blank\">See Playwright docs</a>.",
+            "type": "string",
+            "editor": "select",
+            "enum": [
+                "networkidle",
+                "load",
+                "domcontentloaded"
+            ],
+            "prefill": "networkidle",
+            "default": "networkidle"
+        },
+        "preNavigationHooks": {
+            "sectionCaption": "Advanced configuration",
+            "title": "Pre-navigation hooks",
+            "type": "string",
+            "description": "Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to navigate.",
+            "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept two arguments: the \"crawlingContext\" object\n// and \"gotoOptions\".\n[\n    async (crawlingContext, gotoOptions) => {\n        const { page } = crawlingContext;\n        // ...\n    },\n]",
+            "editor": "javascript"
+        },
+        "postNavigationHooks": {
+            "title": "Post-navigation hooks",
+            "type": "string",
+            "description": "Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.",
+            "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept a single argument: the \"crawlingContext\" object.\n[\n    async (crawlingContext) => {\n        const { page } = crawlingContext;\n        // ...\n    },\n]",
+            "editor": "javascript"
+        },
+        "debugLog": {
+            "title": "Debug log",
+            "type": "boolean",
+            "description": "Debug messages will be included in the log. Use <code>context.log.debug('message')</code> to log your own debug messages.",
+            "default": false,
+            "groupCaption": "Enable logs",
+            "groupDescription": "Logs settings"
+        },
+        "browserLog": {
+            "title": "Browser log",
+            "type": "boolean",
+            "description": "Console messages from the Browser will be included in the log. This may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.",
+            "default": false
+        },
+        "customData": {
+            "title": "Custom data",
+            "type": "object",
+            "description": "This object will be available on pageFunction's context as customData.",
+            "default": {},
+            "prefill": {},
+            "editor": "json"
+        },
+        "datasetName": {
+            "title": "Dataset name",
+            "type": "string",
+            "description": "Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.",
+            "editor": "textfield"
+        },
+        "keyValueStoreName": {
+            "title": "Key-value store name",
+            "type": "string",
+            "description": "Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.",
+            "editor": "textfield"
+        },
+        "requestQueueName": {
+            "title": "Request queue name",
+            "type": "string",
+            "description": "Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.",
+            "editor": "textfield"
+        }
+    },
+    "required": []
+}