From 39ab439f9dba51155e30744cb105042352b656eb Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 17 Oct 2024 09:46:51 +0000 Subject: [PATCH 1/6] Move rewriting stuff from warc2zim to zimscraperlib --- .github/workflows/Publish.yaml | 134 +- .github/workflows/PublishDev.yaml | 47 + .github/workflows/QA.yaml | 78 +- .github/workflows/Tests.yaml | 108 +- .gitignore | 13 + .pre-commit-config.yaml | 55 +- javascript/.prettierignore | 3 + javascript/.prettierrc.json | 3 + javascript/eslint.config.js | 7 + javascript/package.json | 49 + javascript/rollup.config.js | 39 + javascript/src/wombatSetup.js | 310 +++ javascript/test/wombatSetup.js | 42 + javascript/test/wombatUrlRewriting.js | 1180 +++++++++ javascript/yarn.lock | 2243 +++++++++++++++++ openzim.toml | 15 + pyproject.toml | 39 +- rules/generate_rules.py | 180 ++ rules/rules.yaml | 213 ++ src/zimscraperlib/image/optimization.py | 2 +- src/zimscraperlib/rewriting/__init__.py | 0 src/zimscraperlib/rewriting/css.py | 221 ++ src/zimscraperlib/rewriting/html.py | 712 ++++++ src/zimscraperlib/rewriting/js.py | 315 +++ src/zimscraperlib/rewriting/rx_replacer.py | 146 ++ .../rewriting/statics/__wb_module_decl.js | 36 + .../rewriting/templates/head_insert.html | 30 + src/zimscraperlib/rewriting/url_rewriting.py | 424 ++++ src/zimscraperlib/zim/_libkiwix.py | 3 +- src/zimscraperlib/zim/creator.py | 2 +- src/zimscraperlib/zim/items.py | 4 +- src/zimscraperlib/zim/metadata.py | 2 +- src/zimscraperlib/zim/providers.py | 2 +- tests/rewriting/__init__.py | 0 tests/rewriting/conftest.py | 100 + tests/rewriting/test_css_rewriting.py | 218 ++ tests/rewriting/test_html_rewriting.py | 1555 ++++++++++++ tests/rewriting/test_js_rewriting.py | 402 +++ tests/rewriting/test_rx_replacer.py | 107 + tests/rewriting/test_url_rewriting.py | 805 ++++++ tests/rewriting/utils.py | 35 + tests/zim/test_zim_creator.py | 7 +- yarn.lock | 248 ++ 43 files changed, 10056 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/PublishDev.yaml create mode 100644 javascript/.prettierignore create mode 100644 javascript/.prettierrc.json create mode 100644 javascript/eslint.config.js create mode 100644 javascript/package.json create mode 100644 javascript/rollup.config.js create mode 100644 javascript/src/wombatSetup.js create mode 100644 javascript/test/wombatSetup.js create mode 100644 javascript/test/wombatUrlRewriting.js create mode 100644 javascript/yarn.lock create mode 100644 openzim.toml create mode 100644 rules/generate_rules.py create mode 100644 rules/rules.yaml create mode 100644 src/zimscraperlib/rewriting/__init__.py create mode 100644 src/zimscraperlib/rewriting/css.py create mode 100644 src/zimscraperlib/rewriting/html.py create mode 100644 src/zimscraperlib/rewriting/js.py create mode 100644 src/zimscraperlib/rewriting/rx_replacer.py create mode 100644 src/zimscraperlib/rewriting/statics/__wb_module_decl.js create mode 100644 src/zimscraperlib/rewriting/templates/head_insert.html create mode 100644 src/zimscraperlib/rewriting/url_rewriting.py create mode 100644 tests/rewriting/__init__.py create mode 100644 tests/rewriting/conftest.py create mode 100644 tests/rewriting/test_css_rewriting.py create mode 100644 tests/rewriting/test_html_rewriting.py create mode 100644 tests/rewriting/test_js_rewriting.py create mode 100644 tests/rewriting/test_rx_replacer.py create mode 100644 tests/rewriting/test_url_rewriting.py create mode 100644 tests/rewriting/utils.py create mode 100644 yarn.lock diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml index b7e1859b..b8595e79 100644 --- a/.github/workflows/Publish.yaml +++ b/.github/workflows/Publish.yaml @@ -1,20 +1,101 @@ -name: Build and upload to PyPI +name: Build and publish to PyPI / NPM on: release: types: [published] jobs: - publish: - runs-on: ubuntu-22.04 + generate-rules: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[scripts] + + - name: Generate rules + run: | + python rules/generate_rules.py + + - name: Save rules artifact + uses: actions/upload-artifact@v4 + with: + path: | + src/zimscraperlib/rewriting/rules.py + tests/rewriting/test_fuzzy_rules.py + javascript/src/fuzzyRules.js + javascript/test/fuzzyRules.js + name: rules + retention-days: 1 + + build-js: + runs-on: ubuntu-24.04 + needs: generate-rules + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version-file: 'javascript/package.json' + + - name: Install JS dependencies + run: yarn install + working-directory: javascript + + - name: Build production JS + run: yarn build-prod + working-directory: javascript + + - name: Save wombat-setup artifact + uses: actions/upload-artifact@v4 + with: + path: javascript/dist/wombatSetup.js + name: wombat-setup + retention-days: 1 + + publish-python: + runs-on: ubuntu-24.04 + needs: + - generate-rules # to have proper Python rules files (src and tests) + - build-js # to have proper wombatSetup.js (needs to be included in sdist) permissions: - id-token: write # mandatory for PyPI trusted publishing + id-token: write # mandatory for PyPI trusted publishing steps: - - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules + + - name: Restore wombat-setup artifact + uses: actions/download-artifact@v4 + with: + name: wombat-setup + path: src/zimscraperlib/rewriting/statics/wombatSetup.js - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version-file: pyproject.toml architecture: x64 @@ -24,5 +105,44 @@ jobs: pip install -U pip build python -m build --sdist --wheel - - name: Upload to PyPI + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1.8 +# OPTIONAL PUBLICATION TO NPM, NOT NEEDED BY SCRAPERS IN THE END + +# publish-js: +# runs-on: ubuntu-24.04 +# needs: +# - generate-rules + +# steps: +# - name: Checkout repo +# uses: actions/checkout@v4 + +# - name: Restore rules artifact +# uses: actions/download-artifact@v4 +# with: +# name: rules + +# - name: Setup Node.JS +# uses: actions/setup-node@v4 +# with: +# node-version-file: 'javascript/package.json' +# registry-url: 'https://registry.npmjs.org' # Setup .npmrc file to publish to npm + +# - name: Install JS dependencies +# run: yarn install +# working-directory: javascript + +# - name: Build production JS +# run: yarn build-prod +# working-directory: javascript + +# - name: Build JS package +# run: yarn pack +# working-directory: javascript + +# - name: Publish to NPM +# run: npm publish $(ls *.tgz) --provenance --access public +# working-directory: javascript +# env: +# NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/PublishDev.yaml b/.github/workflows/PublishDev.yaml new file mode 100644 index 00000000..8d3ef01b --- /dev/null +++ b/.github/workflows/PublishDev.yaml @@ -0,0 +1,47 @@ +name: Publish dev wombat-setup + +on: + push: + branches: + - main + +jobs: + publish-dev-wombat-setup: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[scripts] + + - name: Generate rules + run: | + python rules/generate_rules.py + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version-file: 'javascript/package.json' + registry-url: 'https://registry.npmjs.org' + + - name: Install JS dependencies + run: yarn install + working-directory: javascript + + - name: Build production JS + run: yarn build-prod + working-directory: javascript + + - name: Upload wombatSetup.js to dev drive + run: | + curl -f -u "${{ secrets.DEV_DRIVE_WEBDAV_CREDENTIALS }}" -T javascript/dist/wombatSetup.js -sw '%{http_code}' "https://dev.kiwix.org/zimscraperlib/" diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml index 48ccee5a..31064c2f 100644 --- a/.github/workflows/QA.yaml +++ b/.github/workflows/QA.yaml @@ -7,14 +7,54 @@ on: - main jobs: - check-qa: - runs-on: ubuntu-22.04 + generate-rules: + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[scripts] + + - name: Generate rules + run: | + python rules/generate_rules.py + + - name: Save rules artifact + uses: actions/upload-artifact@v4 + with: + path: | + src/zimscraperlib/rewriting/rules.py + tests/rewriting/test_fuzzy_rules.py + javascript/src/fuzzyRules.js + javascript/test/fuzzyRules.js + name: rules + retention-days: 1 + + check-python-qa: + runs-on: ubuntu-24.04 + needs: generate-rules + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules + + - name: Set up Python + uses: actions/setup-python@v5 with: python-version-file: pyproject.toml architecture: x64 @@ -32,3 +72,33 @@ jobs: - name: Check pyright run: inv check-pyright + + check-javascript-qa: + runs-on: ubuntu-24.04 + needs: generate-rules + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version-file: 'javascript/package.json' + + - name: Install JS dependencies + working-directory: javascript + run: yarn install + + - name: Check prettier formatting + working-directory: javascript + run: yarn prettier-check + + - name: Check eslint rules + working-directory: javascript + run: yarn eslint diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 0fd2de44..66e647fc 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -7,23 +7,59 @@ on: - main jobs: - run-tests: - strategy: - matrix: - os: [ubuntu-22.04] - python: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ matrix.os }} + generate-rules: + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + architecture: x64 + + - name: Install dependencies (and project) + run: | + pip install -U pip + pip install -e .[scripts] + + - name: Generate rules + run: | + python rules/generate_rules.py + + - name: Save rules artifact + uses: actions/upload-artifact@v4 + with: + path: | + src/zimscraperlib/rewriting/rules.py + tests/rewriting/test_fuzzy_rules.py + javascript/src/fuzzyRules.js + javascript/test/fuzzyRules.js + name: rules + retention-days: 1 + + run-python-tests: + runs-on: ubuntu-24.04 + needs: generate-rules + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules - name: install ffmpeg and gifsicle run: sudo apt update && sudo apt install ffmpeg gifsicle - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python }} + python-version-file: pyproject.toml architecture: x64 - name: Install dependencies (and project) @@ -35,24 +71,50 @@ jobs: run: inv coverage --args "--runslow --runinstalled -vvv" - name: Upload coverage report to codecov - if: matrix.python == '3.12' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} - build_python: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version-file: pyproject.toml - architecture: x64 - - name: Ensure we can build Python targets run: | pip install -U pip build python3 -m build --sdist --wheel + + run-js-tests: + runs-on: ubuntu-24.04 + needs: generate-rules + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Restore rules artifact + uses: actions/download-artifact@v4 + with: + name: rules + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version-file: 'javascript/package.json' + + - name: Install JS dependencies + run: yarn install + working-directory: javascript + + - name: Run JS tests + working-directory: javascript + run: yarn test + + - name: Ensure we can build development JS + run: yarn build-dev + working-directory: javascript + + - name: Ensure we can build production JS + run: yarn build-prod + working-directory: javascript + + - name: Ensure we can build JS package + run: yarn pack + working-directory: javascript diff --git a/.gitignore b/.gitignore index 288bff6b..15586154 100644 --- a/.gitignore +++ b/.gitignore @@ -252,3 +252,16 @@ $RECYCLE.BIN/ # ignore all vscode, this is not standard configuration in this place .vscode src/libzim-stubs +javascript/node_modules + +# rule files are generated by rules/generate_rules.py +src/zimscraperlib/rewriting/rules.py +tests/rewriting/test_fuzzy_rules.py +javascript/src/fuzzyRules.js +javascript/test/fuzzyRules.js + +# wombatSetup.js is generated with rollup +src/zimscraperlib/rewriting/statics/wombatSetup.js + +# wombat.js is installed from online source +src/zimscraperlib/rewriting/statics/wombat.js diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e527d87f..a3e73b4e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,27 +2,34 @@ # See https://pre-commit.com/hooks.html for more hooks exclude: ^tests/files # these are raw test files, no need to mess with them repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer -- repo: https://github.com/psf/black - rev: "24.4.2" - hooks: - - id: black -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 - hooks: - - id: ruff -- repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.368 - hooks: - - id: pyright - name: pyright (system) - description: 'pyright static type checker' - entry: pyright - language: system - 'types_or': [python, pyi] - require_serial: true - minimum_pre_commit_version: '2.9.2' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - repo: https://github.com/psf/black + rev: '24.10.0' + hooks: + - id: black + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.9 + hooks: + - id: ruff + - repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.368 + hooks: + - id: pyright + name: pyright (system) + description: 'pyright static type checker' + entry: pyright + language: system + 'types_or': [python, pyi] + require_serial: true + minimum_pre_commit_version: '2.9.2' + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.1.0 + hooks: + - id: prettier + args: + - --config + - javascript/.prettierrc.json diff --git a/javascript/.prettierignore b/javascript/.prettierignore new file mode 100644 index 00000000..72af6572 --- /dev/null +++ b/javascript/.prettierignore @@ -0,0 +1,3 @@ +**/*.py +src/fuzzyRules.js +test/fuzzyRules.js diff --git a/javascript/.prettierrc.json b/javascript/.prettierrc.json new file mode 100644 index 00000000..544138be --- /dev/null +++ b/javascript/.prettierrc.json @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} diff --git a/javascript/eslint.config.js b/javascript/eslint.config.js new file mode 100644 index 00000000..9eb14339 --- /dev/null +++ b/javascript/eslint.config.js @@ -0,0 +1,7 @@ +export default [ + { + rules: { + 'prefer-const': 'error', + }, + }, +]; diff --git a/javascript/package.json b/javascript/package.json new file mode 100644 index 00000000..1e71ca25 --- /dev/null +++ b/javascript/package.json @@ -0,0 +1,49 @@ +{ + "name": "@openzim/wombat-setup", + "type": "module", + "version": "0.1.0-dev.0", + "license": "GPL-3.0-or-later", + "author": "openZIM", + "devDependencies": { + "@rollup/plugin-commonjs": "26.0.1", + "@rollup/plugin-node-resolve": "15.2.3", + "@rollup/plugin-strip": "^3.0.4", + "@rollup/plugin-terser": "0.4.4", + "ava": "^6.1.3", + "babel-eslint": "^10.1.0", + "eslint": "9.9.1", + "eslint-config-prettier": "9.1.0", + "prettier": "3.3.3", + "rollup": "4.21.2", + "rollup-plugin-version-injector": "^1.3.3" + }, + "scripts": { + "prettier-check": "prettier . --check", + "prettier-fix": "prettier . --write", + "eslint": "eslint .", + "test": "ava --verbose", + "build-prod": "rollup -c rollup.config.js", + "build-dev": "DEV=1 rollup -c rollup.config.js", + "build-dev-watch": "DEV=1 rollup --watch -c rollup.config.js" + }, + "prettier": { + "singleQuote": true + }, + "ava": { + "concurrency": 1, + "verbose": true, + "serial": true, + "files": [ + "test/*.js" + ], + "sources": [ + "src/**/*" + ] + }, + "dependencies": { + "uri-js": "^4.4.1" + }, + "engines": { + "node": ">=20.0.0 <21.0.0" + } +} diff --git a/javascript/rollup.config.js b/javascript/rollup.config.js new file mode 100644 index 00000000..685f3ad6 --- /dev/null +++ b/javascript/rollup.config.js @@ -0,0 +1,39 @@ +import { nodeResolve } from '@rollup/plugin-node-resolve'; // used to bundle node_modules code +import commonjs from '@rollup/plugin-commonjs'; // used to bundle CommonJS node_modules +import terser from '@rollup/plugin-terser'; // used to minify JS code +import strip from '@rollup/plugin-strip'; +import versionInjector from 'rollup-plugin-version-injector'; + +const noStrict = { + renderChunk(code) { + return code.replace("'use strict';", ''); + }, +}; + +const watchOptions = { + exclude: 'node_modules/**', + chokidar: { + alwaysStat: true, + usePolling: true, + }, +}; + +const plugins = [nodeResolve({ preferBuiltins: false }), commonjs(), noStrict]; +if (!process.env.DEV) { + plugins.push(terser()); + plugins.push(strip()); +} +plugins.push(versionInjector()); // do it last so that it is kept no matter what + +export default { + input: 'src/wombatSetup.js', + output: { + name: 'wombatSetup', + file: 'dist/wombatSetup.js', + sourcemap: false, + format: 'iife', + exports: 'named', + }, + watch: watchOptions, + plugins: plugins, +}; diff --git a/javascript/src/wombatSetup.js b/javascript/src/wombatSetup.js new file mode 100644 index 00000000..d3511db3 --- /dev/null +++ b/javascript/src/wombatSetup.js @@ -0,0 +1,310 @@ +import { fuzzyRules } from './fuzzyRules.js'; +import URI from 'uri-js'; + +// Will be updated by rollup-plugin-version-injector +const VERSION = '[VI]version {version}, built on {date}[/VI]'; +console.info(`Running wombat-setup ${VERSION}`); + +export function applyFuzzyRules(path) { + // Apply fuzzy rules to simplify the ZIM path. First matching rule is applied and + // result is immediately returned + + for (const rule of fuzzyRules) { + const new_path = path.replace(new RegExp(rule.match), rule.replace); + if (new_path != path) { + return new_path; + } + } + return path; +} + +export function hasAlreadyBeenRewritten( + original_absolute_url, + orig_url, + uri, + url, +) { + // Detect (with a heuristic) that the path is most probably already rewritten and + // must be kept as-is. We just need to detect relative links (all statically rewritten + // links are relative) and contains a path including the hostname (which cannot be + // joined with the orig_url since if it includes the hostname, it means it is in + // another hostname than orig_url and will hence go one level too high in the path + // hierarchy, hence working only on ZIM paths / relative links). + // The heurisitic is: + // - the link must be relative and start by going at least one level up + // - the first non relative part of the path (i.e. not . or ..) looks like a hostname + // (i.e. it contains a dot) + // - the relative link, when merged with orig_url, is going exactly one "path level" + // too high in the hierarchy + if (typeof uri.scheme == 'undefined' && url.startsWith('../')) { + const urlParts = url.split('/'); + const original_absolute_url1 = URI.resolve( + orig_url, + urlParts.slice(1).join('/'), + ); + const original_absolute_url2 = URI.resolve( + orig_url, + urlParts.slice(2).join('/'), + ); + // detect that relative link is going exactly one "path level" too high + if ( + original_absolute_url1 == original_absolute_url && + original_absolute_url2 != original_absolute_url + ) { + const firstNonRelativePart = urlParts.find((urlPart) => urlPart !== '..'); + // detect that first non relative part of the path looks like a hostname + if (firstNonRelativePart.indexOf('.') > -1) { + // if all 3 conditions are true, then we assume it has already been rewritten + return true; + } + } + } + // otherwise we don't know and assume it can be safely rewritten + return false; +} + +function removeSubsequentSlashes(value) { + // Remove all successive occurrences of a slash `/` in a given string + // E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into `value` + return value.replace(/\/\/+/g, '/'); +} + +export function urlRewriteFunction( + current_url, // The current (real) url we are on, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/index.html + orig_host, // The host of the original url, e.g. www.example.com + orig_scheme, // The scheme of the original url, e.g. https + orig_url, // The original url, e.g. https://www.example.com/index.html + prefix, // The (absolute) prefix to add to all our urls (from where we are served), e.g. http://library.kiwix.org/content/myzim_yyyy-mm/ + url, // first argument passed by wombat.JS at each invocation, current url to rewrite, e.g. http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/image.png + useRel, + mod, + doc, // last argument passed by wombat.JS at each invocation +) { + if (!url) return url; + + // Transform URL which might be an object (detected on Chromium browsers at least) + url = String(url); + + // Special stuff which is not really a URI but exists in the wild + if (['#', '{', '*'].includes(url.substring(0, 1))) return url; + + // If URI scheme is defined but not http or https, we have to not rewrite the URL + const uri = URI.parse(url); + if ( + typeof uri.scheme !== 'undefined' && + !['http', 'https'].includes(uri.scheme) + ) + return url; + + // If url starts with prefix, we need to remove this prefix before applying usual + // rewrite rules + if (url.startsWith(prefix)) { + url = uri.scheme + '://' + url.substring(prefix.length); + } + + // This is a hack to detect improper URL encoding ; proper detection should be + // possible with chardet or other alternatives but did not worked so far ; we hence + // take benefit of the error below to detect improper URL encoding + // When improper URL encoding is detected, we try to encode URL as a best-effort; + // 'best-effort', because if some part of the URL is encoded and another part is not, + // this will fail ... but this is a weird edge case anyway + try { + decodeURIComponent(URI.parse(url).path); + } catch (e) { + url = encodeURI(url); + } + + // Compute the absolute URI, just like the browser would have resolved it hopefully + // We need to use the original URL for that to properly detect the hostname when + // present ; current URL does not allow to do it easily + const original_absolute_url = URI.resolve(orig_url, url); + + // Detect if url has probably already been rewritten and return as-is in such a case + if (hasAlreadyBeenRewritten(original_absolute_url, orig_url, uri, url)) { + return url; + } + + // Detect (with a heuristic) that the path is most probably already rewritten and + // must be kept as-is. We just need to detect relative links (all statically rewritten + // links are relative) and contains a path including the hostname (which cannot be + // joined with the orig_url since if it includes the hostname, it means it is in + // another hostname than orig_url and will hence go one level too high in the path + // hierarchy, hence working only on ZIM paths / relative links). + // The heurisitic is: + // - the link must be relative and start by going at least one level up + // - the first non relative part of the path (i.e. not . or ..) looks like a hostname + // (i.e. it contains a dot) + // - the relative link, when merged with orig_url, is going exactly one "path level" + // too high in the hierarchy + if (typeof uri.scheme == 'undefined' && url.startsWith('../')) { + const urlParts = url.split('/'); + const original_absolute_url1 = URI.resolve( + orig_url, + urlParts.slice(1).join('/'), + ); + const original_absolute_url2 = URI.resolve( + orig_url, + urlParts.slice(2).join('/'), + ); + // detect that relative link is going exactly one "path level" too high + if ( + original_absolute_url1 == original_absolute_url && + original_absolute_url2 != original_absolute_url + ) { + const firstNonRelativePart = urlParts.find((urlPart) => urlPart !== '..'); + // detect that first non relative part of the path looks like a hostname + if (firstNonRelativePart.indexOf('.') > -1) { + // if all 3 conditions are true, then we do not rewrite the link at all, + // otherwise we continue with normal rewritting + return url; + } + } + } + + // We now have to transform this absolute URI into a normalized ZIM path entry + const absolute_url_parts = URI.parse(original_absolute_url); + + // Let's first compute the decoded host + const serialized_host = URI.serialize( + URI.parse('http://' + absolute_url_parts.host), // fake URI to benefit from decoding + { iri: true }, // decode potentially puny-encoded host + ); + const decoded_host = serialized_host.substring(7, serialized_host.length - 1); + + // And the decoded path, only exception is that an empty path must resolve to '/' path + // (our convention, just like in Python) + const decoded_path = + !absolute_url_parts.path || absolute_url_parts.path.length === 0 + ? '/' + : decodeURIComponent(absolute_url_parts.path); + + // And the decoded query, only exception is that + sign must resolve to ' ' to avoid + // confusion (our convention, just like in Python) + const decoded_query = + !absolute_url_parts.query || absolute_url_parts.query.length === 0 + ? '' + : '?' + decodeURIComponent(absolute_url_parts.query).replaceAll('+', ' '); + + // combine all decoded parts to get the ZIM path + const zimPath = + decoded_host + removeSubsequentSlashes(decoded_path + decoded_query); + + // apply the fuzzy rules to the ZIM path + const fuzzifiedPath = applyFuzzyRules(zimPath); + + // Reencode everything but '/' (we decode it afterwards for simplicity) + const finalUrl = + prefix + encodeURIComponent(fuzzifiedPath).replaceAll('%2F', '/'); + + console.debug( + 'urlRewriten:\n\t- current_url: ' + + current_url + + '\n\t- orig_host: ' + + orig_host + + '\n\t- orig_scheme: ' + + orig_scheme + + '\n\t- orig_url: ' + + orig_url + + '\n\t- prefix: ' + + prefix + + '\n\t- url: ' + + url + + '\n\t- useRel: ' + + useRel + + '\n\t- mod: ' + + mod + + '\n\t- doc: ' + + doc + + '\n\t- finalUrl: ' + + finalUrl.toString() + + '\n\t', + ); + + return finalUrl; +} + +export function getWombatInfo( + current_url, // The current (real) url we are on + orig_host, // The host of the original url + orig_scheme, // The scheme of the original url + orig_url, // The original url + prefix, // The (absolute) prefix to add to all our urls (from where we are served)) +) { + return { + // The rewrite function used to rewrite our urls. + rewrite_function: (url, useRel, mod, doc) => + urlRewriteFunction( + current_url, + orig_host, + orig_scheme, + orig_url, + prefix, + url, + useRel, + mod, + doc, + ), + + // Seems to be used only to send message to. We don't care ? + top_url: current_url, + + // Seems to be used to generate url for blobUrl returned by SW. + // We don't care (?) + url: orig_url, + + // Use to timestamp message send to top frame. Don't care + timestamp: '', + + // Use to send message to top frame and in default rewrite url function. Don't care + request_ts: '', + + // The url on which we are served. + prefix: prefix, + + // The default mod to use. + mod: '', + + // Use to detect if we are framed (and send message to top frame ?) + is_framed: false, + + // ?? + is_live: false, + + // Never used ? + coll: '', + + // Set wombat if is proxy mode (we are not) + proxy_magic: '', + + // This is the prefix on which we have stored our static files (needed by wombat). + // Must not conflict with other url served. + // Will be used by wombat to not rewrite back the url + static_prefix: prefix + '_zim_static/', + + wombat_ts: '', + + // A delay in sec to apply to all js time (`Date.now()`, ...) + wombat_sec: 0, + + // The scheme of the original url + wombat_scheme: orig_scheme, + + // The host of the original url + wombat_host: orig_host, + + // Extra options ? + wombat_opts: {}, + + // ? + enable_auto_fetch: true, + convert_post_to_get: true, + target_frame: '___wb_replay_top_frame', + isSW: true, + }; +} + +export default { + applyFuzzyRules: applyFuzzyRules, + urlRewriteFunction: urlRewriteFunction, + getWombatInfo: getWombatInfo, +}; diff --git a/javascript/test/wombatSetup.js b/javascript/test/wombatSetup.js new file mode 100644 index 00000000..444d0428 --- /dev/null +++ b/javascript/test/wombatSetup.js @@ -0,0 +1,42 @@ +import test from 'ava'; + +import utils from '../src/wombatSetup.js'; + +test.beforeEach((t) => { + t.context.prefix = 'http://library.kiwix.org/content/myzim_yyyy-mm/'; + t.context.originalHost = 'www.example.com'; + t.context.originalScheme = 'https'; +}); + +test('nominalWbInfo', (t) => { + const path = 'path1/resource1.js'; + const originalUrl = + t.context.originalScheme + '://' + t.context.originalHost + '/' + path; + const wmInfo = utils.getWombatInfo( + t.context.prefix + path, + t.context.originalHost, + t.context.originalScheme, + originalUrl, + t.context.prefix, + ); + t.is(wmInfo.coll, ''); + t.is(wmInfo.convert_post_to_get, true); + t.is(wmInfo.enable_auto_fetch, true); + t.is(wmInfo.isSW, true); + t.is(wmInfo.is_framed, false); + t.is(wmInfo.is_live, false); + t.is(wmInfo.mod, ''); + t.is(wmInfo.prefix, t.context.prefix); + t.is(wmInfo.proxy_magic, ''); + t.is(wmInfo.request_ts, ''); + t.is(wmInfo.static_prefix, t.context.prefix + '_zim_static/'); + t.is(wmInfo.target_frame, '___wb_replay_top_frame'); + t.is(wmInfo.timestamp, ''); + t.is(wmInfo.top_url, t.context.prefix + path); + t.is(wmInfo.url, originalUrl); + t.is(wmInfo.wombat_host, t.context.originalHost); + t.deepEqual(wmInfo.wombat_opts, {}); + t.is(wmInfo.wombat_scheme, t.context.originalScheme); + t.is(wmInfo.wombat_sec, 0); + t.is(wmInfo.wombat_ts, ''); +}); diff --git a/javascript/test/wombatUrlRewriting.js b/javascript/test/wombatUrlRewriting.js new file mode 100644 index 00000000..e047f842 --- /dev/null +++ b/javascript/test/wombatUrlRewriting.js @@ -0,0 +1,1180 @@ +import test from 'ava'; + +import { urlRewriteFunction } from '../src/wombatSetup.js'; + +test.beforeEach((t) => { + t.context.prefix = 'http://library.kiwix.org/content/myzim_yyyy-mm/'; + t.context.originalHost = 'www.example.com'; + t.context.originalScheme = 'https'; + t.context.documentPath = '/path1/resource1.html'; + t.context.originalUrl = + t.context.originalScheme + + '://' + + t.context.originalHost + + t.context.documentPath; + t.context.currentUrl = + t.context.prefix + t.context.originalHost + t.context.documentPath; +}); + +test('undefined', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + undefined, + undefined, + undefined, + undefined, + ), + undefined, + ); +}); + +test('simpleContentCompleteUrl', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('simpleContentFullUrl', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://user:password@www.example.com:8888/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('simpleContentNoScheme', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '//www.example.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('simpleContentEmptyPath', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/', + ); +}); + +test('simpleContentNoPath', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/', + ); +}); + +test('contentWithSpecialCharsNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/contént.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%C3%A9nt.txt', + ); +}); + +test('contentWithUTF8CharsNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont🎁nt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%F0%9F%8E%81nt.txt', + ); +}); + +test('contentWithSpecialCharsEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%C3%A9nt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%C3%A9nt.txt', + ); +}); + +test('contentWithSpaceNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont nt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%20nt.txt', + ); +}); + +test('contentWithPlusNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont+nt.txt', // + is not unreserved, it must be encoded to avoid confusion + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%2Bnt.txt', + ); +}); + +test('contentWithTilde', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont~nt.txt', // ~ is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont~nt.txt', + ); +}); + +test('contentWithHyphen', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont-nt.txt', // - is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont-nt.txt', + ); +}); + +test('contentWithUnderscore', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont_nt.txt', // _ is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont_nt.txt', + ); +}); + +test('contentWithEncodedTilde', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%7Ent.txt', // ~ is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont~nt.txt', + ); +}); + +test('contentWithEncodedApostrophe', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%27nt.txt', // ' is reserved, but it is not encoded in JS + undefined, + undefined, + undefined, + ), + "http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont'nt.txt", + ); +}); + +test('contentWithEncodedExclamationMark', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%21nt.txt', // ! is reserved, but it is not encoded in JS + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont!nt.txt', + ); +}); + +test('contentWithEncodedQuestionMark', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%3Fnt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt', + ); +}); + +test('contentWithEncodedQuestionMarkAndQueryParam', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%3Fnt.txt?query=value', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%3Fnt.txt%3Fquery%3Dvalue', + ); +}); + +test('contentWithEncodedStar', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%2Ant.txt', // * is reserved, but it is not encoded in JS + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont*nt.txt', + ); +}); + +test('contentWithEncodedParentheses', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%28%29nt.txt', // ( and ) are reserved, but they are not encoded in JS + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont()nt.txt', + ); +}); + +test('contentWithEncodedHyphen1', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%2Dnt.txt', // - is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont-nt.txt', + ); +}); + +test('contentWithEncodedHyphen2', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/co%25nt%2Dnt.txt', // - is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/co%25nt-nt.txt', + ); +}); + +test('contentWithEncodedUnderscore', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/cont%5Fnt.txt', // _ is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont_nt.txt', + ); +}); + +test('contentWithEncodedPeriod', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content%2Etxt', // . is unreserved, it must be not be encoded + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('contentWithSimpleQueryString', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt?query=value', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dvalue', + ); +}); + +test('contentWithQueryValueEqualSign', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt?query=val%3Deue', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dval%3Deue', + ); +}); + +test('contentWithQueryValuePercentSign', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt?query=val%25eue', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dval%25eue', + ); +}); + +test('contentWithQueryParamPercentSign', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt?que%25ry=valeue', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fque%25ry%3Dvaleue', + ); +}); + +test('contentWithQueryParamPlusSign', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/content.txt?param=val+ue', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fparam%3Dval%20ue', + ); +}); + +test('fqdnWithSpecialCharsEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.xn--xample-9ua.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.%C3%A9xample.com/javascript/content.txt', + ); +}); + +test('fqdnWithSpecialCharsNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.éxample.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.%C3%A9xample.com/javascript/content.txt', + ); +}); + +test('fqdnWithSpecialCharsEncodedContentWithSpecialCharsEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.xn--xample-9ua.com/javascript/cont%C3%A9nt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.%C3%A9xample.com/javascript/cont%C3%A9nt.txt', + ); +}); + +test('relSimpleContent1', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('relSimpleContent2', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + './javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/javascript/content.txt', + ); +}); + +test('relGoingUperThanHost', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../../../../javascript/content.txt', // this is too many .. ; at this stage it means host home folder + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('relSimpleContent3', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('mailto', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'mailto:bob@example.com', + undefined, + undefined, + undefined, + ), + 'mailto:bob@example.com', + ); +}); + +test('data', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'data:bob@example.com', + undefined, + undefined, + undefined, + ), + 'data:bob@example.com', + ); +}); + +test('tel', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'tel:+1.43.88.999.999', + undefined, + undefined, + undefined, + ), + 'tel:+1.43.88.999.999', + ); +}); + +test('ftp', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'ftp://www.example.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'ftp://www.example.com/javascript/content.txt', + ); +}); + +test('blob', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'blob:exemple.com/url', + undefined, + undefined, + undefined, + ), + 'blob:exemple.com/url', + ); +}); + +test('customprotocol', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'customprotocol:exemple.com/url', + undefined, + undefined, + undefined, + ), + 'customprotocol:exemple.com/url', + ); +}); + +test('anchor', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '#anchor', + undefined, + undefined, + undefined, + ), + '#anchor', + ); +}); + +test('star', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '*wtf', + undefined, + undefined, + undefined, + ), + '*wtf', + ); +}); + +test('mustache', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '{wtf', + undefined, + undefined, + undefined, + ), + '{wtf', + ); +}); + +test('youtubeFuzzyNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'www.youtube.com/get_video_info?video_id=123ah', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D123ah', + ); +}); + +test('youtubeFuzzyEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'www.youtube.com/get_video_info?video_id=12%3D3ah', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/path1/youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D12%3D3ah', + ); +}); + +test('alreadyRewritenUrlSimple', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +test('alreadyRewritenUrlSpecialCharsNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/contént.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%C3%A9nt.txt', + ); +}); + +test('alreadyRewritenUrlUTF8CharsNotEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont🎁nt.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/cont%F0%9F%8E%81nt.txt', + ); +}); + +test('simpleContentDomainNameInPath1', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/www.example.com/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/www.example.com/content.txt', + ); +}); + +test('simpleContentDomainNameInPath2', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'https://www.example.com/javascript/www.example.com', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/www.example.com', + ); +}); + +// URL has already been statically rewritten and originally had a query parameter +test('relAlreadyEncoded', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../javascript/content.txt%3Fquery%3Dvalue', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt%3Fquery%3Dvalue', + ); +}); + +// this is an edge case where the URL has already been statically rewritten and is located +// on a different domain name => we do not touch it at all +test('relAnotherHostAlreadyRewritten', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../../anotherhost.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + '../../anotherhost.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL has already been statically rewritten and is located +// on a different domain name => we do not touch it at all +test('relAnotherHostAlreadyRewrittenRootPath', (t) => { + const documentPath = '/'; + const originalUrl = + t.context.originalScheme + '://' + t.context.originalHost + documentPath; + const currentUrl = t.context.prefix + t.context.originalHost + documentPath; + t.is( + urlRewriteFunction( + currentUrl, + t.context.originalHost, + t.context.originalScheme, + originalUrl, + t.context.prefix, + '../anotherhost.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + '../anotherhost.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL has already been statically rewritten and is located +// on a different domain name => we do not touch it at all +test('relAnotherHostAlreadyRewrittenEmptyPath', (t) => { + const documentPath = ''; + const originalUrl = + t.context.originalScheme + '://' + t.context.originalHost + documentPath; + const currentUrl = t.context.prefix + t.context.originalHost + documentPath; + t.is( + urlRewriteFunction( + currentUrl, + t.context.originalHost, + t.context.originalScheme, + originalUrl, + t.context.prefix, + '../anotherhost.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + '../anotherhost.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL has already been statically rewritten and is located +// on a different fuzzified domain name => we do not touch it at all +test('relAnotherFuzzifiedHostAlreadyRewritten', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../../youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D123ah', + undefined, + undefined, + undefined, + ), + '../../youtube.fuzzy.replayweb.page/get_video_info%3Fvideo_id%3D123ah', + ); +}); + +// this is an edge case where the URL might looks like it has already been statically +// rewritten since it is going too up exactly by one level but it does not looks like a +// hostname at all => we rewrite it again +test('relTooUpNotLookingLikeAHostname', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../../javascript/content.txt', // this is too many .. ; at this stage it means host home folder + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL might looks like it has already been statically +// rewritten since it is going too up exactly by one level but it does not looks like a +// hostname at all => we rewrite it again +test('relTooUpNotLookingLikeAHostnameRootPath', (t) => { + const documentPath = '/'; + const originalUrl = + t.context.originalScheme + '://' + t.context.originalHost + documentPath; + const currentUrl = t.context.prefix + t.context.originalHost + documentPath; + t.is( + urlRewriteFunction( + currentUrl, + t.context.originalHost, + t.context.originalScheme, + originalUrl, + t.context.prefix, + '../javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL might looks like it has already been statically +// rewritten since it is going too up exactly by one level but it does not looks like a +// hostname at all => we rewrite it again +test('relTooUpNotLookingLikeAHostnameEmptyPath', (t) => { + const documentPath = ''; + const originalUrl = + t.context.originalScheme + '://' + t.context.originalHost + documentPath; + const currentUrl = t.context.prefix + t.context.originalHost + documentPath; + t.is( + urlRewriteFunction( + currentUrl, + t.context.originalHost, + t.context.originalScheme, + originalUrl, + t.context.prefix, + '../javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL might looks like it has already been statically +// rewritten and is located on a different domain name but it is going to way up in the +// hierarchy so it is most probably not really rewritten yet => we rewrite it again +test('relNotReallyAnotherHostAlreadyRewrittenTooUp', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../../../anotherhost.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/anotherhost.com/javascript/content.txt', + ); +}); + +// this is an edge case where the URL might looks like it has already been statically +// rewritten and is located on a different domain name but it is going no enough way up +// in the hierarchy so it is most probably not really rewritten yet => we rewrite it +// again +test('relNotReallyAnotherHostAlreadyRewrittenNotUpEnough', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + '../anotherhost.com/javascript/content.txt', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/www.example.com/anotherhost.com/javascript/content.txt', + ); +}); + +test('doubleSlash1', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://example.com/some/path/http://example.com//some/path', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/path/http%3A/example.com/some/path', + ); +}); + +test('doubleSlash2', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://example.com/some/pa?th/http://example.com//some/path', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/some/pa%3Fth/http%3A/example.com/some/path', + ); +}); + +test('doubleSlash3', (t) => { + t.is( + urlRewriteFunction( + t.context.currentUrl, + t.context.originalHost, + t.context.originalScheme, + t.context.originalUrl, + t.context.prefix, + 'http://example.com/so?me/pa?th/http://example.com//some/path', + undefined, + undefined, + undefined, + ), + 'http://library.kiwix.org/content/myzim_yyyy-mm/example.com/so%3Fme/pa%3Fth/http%3A/example.com/some/path', + ); +}); diff --git a/javascript/yarn.lock b/javascript/yarn.lock new file mode 100644 index 00000000..1de7f387 --- /dev/null +++ b/javascript/yarn.lock @@ -0,0 +1,2243 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.25.7.tgz#438f2c524071531d643c6f0188e1e28f130cebc7" + integrity sha512-0xZJFNE5XMpENsgfHYTw8FbX4kv53mFLn2i3XPoq69LyhYSCBJtitaHx9QnsVTrsogI4Z3+HtEfZ2/GFPOtf5g== + dependencies: + "@babel/highlight" "^7.25.7" + picocolors "^1.0.0" + +"@babel/generator@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.25.7.tgz#de86acbeb975a3e11ee92dd52223e6b03b479c56" + integrity sha512-5Dqpl5fyV9pIAD62yK9P7fcA768uVPUyrQmqpqstHWgMma4feF1x/oFysBCVZLY5wJ2GkMUCdsNDnGZrPoR6rA== + dependencies: + "@babel/types" "^7.25.7" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^3.0.2" + +"@babel/helper-string-parser@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.25.7.tgz#d50e8d37b1176207b4fe9acedec386c565a44a54" + integrity sha512-CbkjYdsJNHFk8uqpEkpCvRs3YRp9tY6FmFY7wLMSYuGYkrdUi7r2lc4/wqsvlHoMznX3WJ9IP8giGPq68T/Y6g== + +"@babel/helper-validator-identifier@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.7.tgz#77b7f60c40b15c97df735b38a66ba1d7c3e93da5" + integrity sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg== + +"@babel/highlight@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.25.7.tgz#20383b5f442aa606e7b5e3043b0b1aafe9f37de5" + integrity sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw== + dependencies: + "@babel/helper-validator-identifier" "^7.25.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.25.7", "@babel/parser@^7.7.0": + version "7.25.8" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.25.8.tgz#f6aaf38e80c36129460c1657c0762db584c9d5e2" + integrity sha512-HcttkxzdPucv3nNFmfOOMfFf64KgdJVqm1KaCm25dPGMLElo9nsLvXeJECQg8UzPuBGLyTSA0ZzqCtDSzKTEoQ== + dependencies: + "@babel/types" "^7.25.8" + +"@babel/template@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.7.tgz#27f69ce382855d915b14ab0fe5fb4cbf88fa0769" + integrity sha512-wRwtAgI3bAS+JGU2upWNL9lSlDcRCqD05BZ1n3X2ONLH1WilFP6O1otQjeMK/1g0pvYcXC7b/qVUB1keofjtZA== + dependencies: + "@babel/code-frame" "^7.25.7" + "@babel/parser" "^7.25.7" + "@babel/types" "^7.25.7" + +"@babel/traverse@^7.7.0": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.25.7.tgz#83e367619be1cab8e4f2892ef30ba04c26a40fa8" + integrity sha512-jatJPT1Zjqvh/1FyJs6qAHL+Dzb7sTb+xr7Q+gM1b+1oBsMsQQ4FkVKb6dFlJvLlVssqkRzV05Jzervt9yhnzg== + dependencies: + "@babel/code-frame" "^7.25.7" + "@babel/generator" "^7.25.7" + "@babel/parser" "^7.25.7" + "@babel/template" "^7.25.7" + "@babel/types" "^7.25.7" + debug "^4.3.1" + globals "^11.1.0" + +"@babel/types@^7.25.7", "@babel/types@^7.25.8", "@babel/types@^7.7.0": + version "7.25.8" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.25.8.tgz#5cf6037258e8a9bcad533f4979025140cb9993e1" + integrity sha512-JWtuCu8VQsMladxVz/P4HzHUGCAwpuqacmowgXFs5XjxIgKuNjnLokQzuVjlTvIzODaDmpjT3oxcC48vyk9EWg== + dependencies: + "@babel/helper-string-parser" "^7.25.7" + "@babel/helper-validator-identifier" "^7.25.7" + to-fast-properties "^2.0.0" + +"@eslint-community/eslint-utils@^4.2.0": + version "4.4.0" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" + integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== + dependencies: + eslint-visitor-keys "^3.3.0" + +"@eslint-community/regexpp@^4.11.0": + version "4.11.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.1.tgz#a547badfc719eb3e5f4b556325e542fbe9d7a18f" + integrity sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q== + +"@eslint/config-array@^0.18.0": + version "0.18.0" + resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.18.0.tgz#37d8fe656e0d5e3dbaea7758ea56540867fd074d" + integrity sha512-fTxvnS1sRMu3+JjXwJG0j/i4RT9u4qJ+lqS/yCGap4lH4zZGzQ7tu+xZqQmcMZq5OBZDL4QRxQzRjkWcGt8IVw== + dependencies: + "@eslint/object-schema" "^2.1.4" + debug "^4.3.1" + minimatch "^3.1.2" + +"@eslint/eslintrc@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-3.1.0.tgz#dbd3482bfd91efa663cbe7aa1f506839868207b6" + integrity sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ== + dependencies: + ajv "^6.12.4" + debug "^4.3.2" + espree "^10.0.1" + globals "^14.0.0" + ignore "^5.2.0" + import-fresh "^3.2.1" + js-yaml "^4.1.0" + minimatch "^3.1.2" + strip-json-comments "^3.1.1" + +"@eslint/js@9.9.1": + version "9.9.1" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.9.1.tgz#4a97e85e982099d6c7ee8410aacb55adaa576f06" + integrity sha512-xIDQRsfg5hNBqHz04H1R3scSVwmI+KUbqjsQKHKQ1DAUSaUjYPReZZmS/5PNiKu1fUvzDd6H7DEDKACSEhu+TQ== + +"@eslint/object-schema@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/object-schema/-/object-schema-2.1.4.tgz#9e69f8bb4031e11df79e03db09f9dbbae1740843" + integrity sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ== + +"@humanwhocodes/module-importer@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" + integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== + +"@humanwhocodes/retry@^0.3.0": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.3.1.tgz#c72a5c76a9fbaf3488e231b13dc52c0da7bab42a" + integrity sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA== + +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" + +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/source-map@^0.3.3": + version "0.3.6" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.6.tgz#9d71ca886e32502eb9362c9a74a46787c36df81a" + integrity sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14", "@jridgewell/sourcemap-codec@^1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" + integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== + +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + +"@mapbox/node-pre-gyp@^1.0.5": + version "1.0.11" + resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa" + integrity sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ== + dependencies: + detect-libc "^2.0.0" + https-proxy-agent "^5.0.0" + make-dir "^3.1.0" + node-fetch "^2.6.7" + nopt "^5.0.0" + npmlog "^5.0.1" + rimraf "^3.0.2" + semver "^7.3.5" + tar "^6.1.11" + +"@nodelib/fs.scandir@2.1.5": + version "2.1.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" + integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== + dependencies: + "@nodelib/fs.stat" "2.0.5" + run-parallel "^1.1.9" + +"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== + +"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8": + version "1.2.8" + resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" + integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== + dependencies: + "@nodelib/fs.scandir" "2.1.5" + fastq "^1.6.0" + +"@pkgjs/parseargs@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" + integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== + +"@rollup/plugin-commonjs@26.0.1": + version "26.0.1" + resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-26.0.1.tgz#16d4d6e54fa63021249a292b50f27c0b0f1a30d8" + integrity sha512-UnsKoZK6/aGIH6AdkptXhNvhaqftcjq3zZdT+LY5Ftms6JR06nADcDsYp5hTU9E2lbJUEOhdlY5J4DNTneM+jQ== + dependencies: + "@rollup/pluginutils" "^5.0.1" + commondir "^1.0.1" + estree-walker "^2.0.2" + glob "^10.4.1" + is-reference "1.2.1" + magic-string "^0.30.3" + +"@rollup/plugin-node-resolve@15.2.3": + version "15.2.3" + resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.2.3.tgz#e5e0b059bd85ca57489492f295ce88c2d4b0daf9" + integrity sha512-j/lym8nf5E21LwBT4Df1VD6hRO2L2iwUeUmP7litikRsVp1H6NWx20NEp0Y7su+7XGc476GnXXc4kFeZNGmaSQ== + dependencies: + "@rollup/pluginutils" "^5.0.1" + "@types/resolve" "1.20.2" + deepmerge "^4.2.2" + is-builtin-module "^3.2.1" + is-module "^1.0.0" + resolve "^1.22.1" + +"@rollup/plugin-strip@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@rollup/plugin-strip/-/plugin-strip-3.0.4.tgz#ad623cc18cf305b484f8bf38fde4ae855c1229e4" + integrity sha512-LDRV49ZaavxUo2YoKKMQjCxzCxugu1rCPQa0lDYBOWLj6vtzBMr8DcoJjsmg+s450RbKbe3qI9ZLaSO+O1oNbg== + dependencies: + "@rollup/pluginutils" "^5.0.1" + estree-walker "^2.0.2" + magic-string "^0.30.3" + +"@rollup/plugin-terser@0.4.4": + version "0.4.4" + resolved "https://registry.yarnpkg.com/@rollup/plugin-terser/-/plugin-terser-0.4.4.tgz#15dffdb3f73f121aa4fbb37e7ca6be9aeea91962" + integrity sha512-XHeJC5Bgvs8LfukDwWZp7yeqin6ns8RTl2B9avbejt6tZqsqvVoWI7ZTQrcNsfKEDWBTnTxM8nMDkO2IFFbd0A== + dependencies: + serialize-javascript "^6.0.1" + smob "^1.0.0" + terser "^5.17.4" + +"@rollup/pluginutils@^4.0.0": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d" + integrity sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ== + dependencies: + estree-walker "^2.0.1" + picomatch "^2.2.2" + +"@rollup/pluginutils@^5.0.1": + version "5.1.2" + resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-5.1.2.tgz#d3bc9f0fea4fd4086aaac6aa102f3fa587ce8bd9" + integrity sha512-/FIdS3PyZ39bjZlwqFnWqCOVnW7o963LtKMwQOD0NhQqw22gSr2YY1afu3FxRip4ZCZNsD5jq6Aaz6QV3D/Njw== + dependencies: + "@types/estree" "^1.0.0" + estree-walker "^2.0.2" + picomatch "^2.3.1" + +"@rollup/rollup-android-arm-eabi@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.2.tgz#0412834dc423d1ff7be4cb1fc13a86a0cd262c11" + integrity sha512-fSuPrt0ZO8uXeS+xP3b+yYTCBUd05MoSp2N/MFOgjhhUhMmchXlpTQrTpI8T+YAwAQuK7MafsCOxW7VrPMrJcg== + +"@rollup/rollup-android-arm64@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.2.tgz#baf1a014b13654f3b9e835388df9caf8c35389cb" + integrity sha512-xGU5ZQmPlsjQS6tzTTGwMsnKUtu0WVbl0hYpTPauvbRAnmIvpInhJtgjj3mcuJpEiuUw4v1s4BimkdfDWlh7gA== + +"@rollup/rollup-darwin-arm64@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.2.tgz#0a2c364e775acdf1172fe3327662eec7c46e55b1" + integrity sha512-99AhQ3/ZMxU7jw34Sq8brzXqWH/bMnf7ZVhvLk9QU2cOepbQSVTns6qoErJmSiAvU3InRqC2RRZ5ovh1KN0d0Q== + +"@rollup/rollup-darwin-x64@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.2.tgz#a972db75890dfab8df0da228c28993220a468c42" + integrity sha512-ZbRaUvw2iN/y37x6dY50D8m2BnDbBjlnMPotDi/qITMJ4sIxNY33HArjikDyakhSv0+ybdUxhWxE6kTI4oX26w== + +"@rollup/rollup-linux-arm-gnueabihf@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.2.tgz#1609d0630ef61109dd19a278353e5176d92e30a1" + integrity sha512-ztRJJMiE8nnU1YFcdbd9BcH6bGWG1z+jP+IPW2oDUAPxPjo9dverIOyXz76m6IPA6udEL12reYeLojzW2cYL7w== + +"@rollup/rollup-linux-arm-musleabihf@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.2.tgz#3c1dca5f160aa2e79e4b20ff6395eab21804f266" + integrity sha512-flOcGHDZajGKYpLV0JNc0VFH361M7rnV1ee+NTeC/BQQ1/0pllYcFmxpagltANYt8FYf9+kL6RSk80Ziwyhr7w== + +"@rollup/rollup-linux-arm64-gnu@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.2.tgz#c2fe376e8b04eafb52a286668a8df7c761470ac7" + integrity sha512-69CF19Kp3TdMopyteO/LJbWufOzqqXzkrv4L2sP8kfMaAQ6iwky7NoXTp7bD6/irKgknDKM0P9E/1l5XxVQAhw== + +"@rollup/rollup-linux-arm64-musl@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.2.tgz#e62a4235f01e0f66dbba587c087ca6db8008ec80" + integrity sha512-48pD/fJkTiHAZTnZwR0VzHrao70/4MlzJrq0ZsILjLW/Ab/1XlVUStYyGt7tdyIiVSlGZbnliqmult/QGA2O2w== + +"@rollup/rollup-linux-powerpc64le-gnu@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.2.tgz#24b3457e75ee9ae5b1c198bd39eea53222a74e54" + integrity sha512-cZdyuInj0ofc7mAQpKcPR2a2iu4YM4FQfuUzCVA2u4HI95lCwzjoPtdWjdpDKyHxI0UO82bLDoOaLfpZ/wviyQ== + +"@rollup/rollup-linux-riscv64-gnu@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.2.tgz#38edfba9620fe2ca8116c97e02bd9f2d606bde09" + integrity sha512-RL56JMT6NwQ0lXIQmMIWr1SW28z4E4pOhRRNqwWZeXpRlykRIlEpSWdsgNWJbYBEWD84eocjSGDu/XxbYeCmwg== + +"@rollup/rollup-linux-s390x-gnu@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.2.tgz#a3bfb8bc5f1e802f8c76cff4a4be2e9f9ac36a18" + integrity sha512-PMxkrWS9z38bCr3rWvDFVGD6sFeZJw4iQlhrup7ReGmfn7Oukrr/zweLhYX6v2/8J6Cep9IEA/SmjXjCmSbrMQ== + +"@rollup/rollup-linux-x64-gnu@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.2.tgz#0dadf34be9199fcdda44b5985a086326344f30ad" + integrity sha512-B90tYAUoLhU22olrafY3JQCFLnT3NglazdwkHyxNDYF/zAxJt5fJUB/yBoWFoIQ7SQj+KLe3iL4BhOMa9fzgpw== + +"@rollup/rollup-linux-x64-musl@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.2.tgz#7b7deddce240400eb87f2406a445061b4fed99a8" + integrity sha512-7twFizNXudESmC9oneLGIUmoHiiLppz/Xs5uJQ4ShvE6234K0VB1/aJYU3f/4g7PhssLGKBVCC37uRkkOi8wjg== + +"@rollup/rollup-win32-arm64-msvc@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.2.tgz#a0ca0c5149c2cfb26fab32e6ba3f16996fbdb504" + integrity sha512-9rRero0E7qTeYf6+rFh3AErTNU1VCQg2mn7CQcI44vNUWM9Ze7MSRS/9RFuSsox+vstRt97+x3sOhEey024FRQ== + +"@rollup/rollup-win32-ia32-msvc@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.2.tgz#aae2886beec3024203dbb5569db3a137bc385f8e" + integrity sha512-5rA4vjlqgrpbFVVHX3qkrCo/fZTj1q0Xxpg+Z7yIo3J2AilW7t2+n6Q8Jrx+4MrYpAnjttTYF8rr7bP46BPzRw== + +"@rollup/rollup-win32-x64-msvc@4.21.2": + version "4.21.2" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.2.tgz#e4291e3c1bc637083f87936c333cdbcad22af63b" + integrity sha512-6UUxd0+SKomjdzuAcp+HAmxw1FlGBnl1v2yEPSabtx4lBfdXHDVsW7+lQkgz9cNFJGY3AWR7+V8P5BqkD9L9nA== + +"@sindresorhus/merge-streams@^2.1.0": + version "2.3.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz#719df7fb41766bc143369eaa0dd56d8dc87c9958" + integrity sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg== + +"@types/estree@*", "@types/estree@^1.0.0": + version "1.0.6" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50" + integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== + +"@types/estree@1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" + integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== + +"@types/resolve@1.20.2": + version "1.20.2" + resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.2.tgz#97d26e00cd4a0423b4af620abecf3e6f442b7975" + integrity sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q== + +"@vercel/nft@^0.26.2": + version "0.26.5" + resolved "https://registry.yarnpkg.com/@vercel/nft/-/nft-0.26.5.tgz#f21e40576b76446851b6cbff79f39a72dab4d6b2" + integrity sha512-NHxohEqad6Ra/r4lGknO52uc/GrWILXAMs1BB4401GTqww0fw1bAqzpG1XHuDO+dprg4GvsD9ZLLSsdo78p9hQ== + dependencies: + "@mapbox/node-pre-gyp" "^1.0.5" + "@rollup/pluginutils" "^4.0.0" + acorn "^8.6.0" + acorn-import-attributes "^1.9.2" + async-sema "^3.1.1" + bindings "^1.4.0" + estree-walker "2.0.2" + glob "^7.1.3" + graceful-fs "^4.2.9" + micromatch "^4.0.2" + node-gyp-build "^4.2.2" + resolve-from "^5.0.0" + +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + +acorn-import-attributes@^1.9.2: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== + +acorn-jsx@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== + +acorn-walk@^8.3.2: + version "8.3.4" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.4.tgz#794dd169c3977edf4ba4ea47583587c5866236b7" + integrity sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g== + dependencies: + acorn "^8.11.0" + +acorn@^8.11.0, acorn@^8.11.3, acorn@^8.12.0, acorn@^8.6.0, acorn@^8.8.2: + version "8.13.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.13.0.tgz#2a30d670818ad16ddd6a35d3842dacec9e5d7ca3" + integrity sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w== + +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + +ajv@^6.12.4: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + +ansi-regex@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.1.0.tgz#95ec409c69619d6cb1b8b34f14b660ef28ebd654" + integrity sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA== + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +ansi-styles@^4.0.0, ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +ansi-styles@^6.0.0, ansi-styles@^6.1.0, ansi-styles@^6.2.1: + version "6.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" + integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + +"aproba@^1.0.3 || ^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc" + integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== + +are-we-there-yet@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz#372e0e7bd279d8e94c653aaa1f67200884bf3e1c" + integrity sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw== + dependencies: + delegates "^1.0.0" + readable-stream "^3.6.0" + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + +array-find-index@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1" + integrity sha512-M1HQyIXcBGtVywBt8WVdim+lrNaK7VHp99Qt5pSNziXznKHViIBbXWtfRTpEFpF/c4FdfxNAsCCwPp5phBYJtw== + +arrgv@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/arrgv/-/arrgv-1.0.2.tgz#025ed55a6a433cad9b604f8112fc4292715a6ec0" + integrity sha512-a4eg4yhp7mmruZDQFqVMlxNRFGi/i1r87pt8SDHy0/I8PqSXoUTlWZRdAZo0VXgvEARcujbtTk8kiZRi1uDGRw== + +arrify@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-3.0.0.tgz#ccdefb8eaf2a1d2ab0da1ca2ce53118759fd46bc" + integrity sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw== + +async-sema@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/async-sema/-/async-sema-3.1.1.tgz#e527c08758a0f8f6f9f15f799a173ff3c40ea808" + integrity sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg== + +ava@^6.1.3: + version "6.1.3" + resolved "https://registry.yarnpkg.com/ava/-/ava-6.1.3.tgz#aed54a4528653c7a62b6d68d0a53608b22a5b1dc" + integrity sha512-tkKbpF1pIiC+q09wNU9OfyTDYZa8yuWvU2up3+lFJ3lr1RmnYh2GBpPwzYUEB0wvTPIUysGjcZLNZr7STDviRA== + dependencies: + "@vercel/nft" "^0.26.2" + acorn "^8.11.3" + acorn-walk "^8.3.2" + ansi-styles "^6.2.1" + arrgv "^1.0.2" + arrify "^3.0.0" + callsites "^4.1.0" + cbor "^9.0.1" + chalk "^5.3.0" + chunkd "^2.0.1" + ci-info "^4.0.0" + ci-parallel-vars "^1.0.1" + cli-truncate "^4.0.0" + code-excerpt "^4.0.0" + common-path-prefix "^3.0.0" + concordance "^5.0.4" + currently-unhandled "^0.4.1" + debug "^4.3.4" + emittery "^1.0.1" + figures "^6.0.1" + globby "^14.0.0" + ignore-by-default "^2.1.0" + indent-string "^5.0.0" + is-plain-object "^5.0.0" + is-promise "^4.0.0" + matcher "^5.0.0" + memoize "^10.0.0" + ms "^2.1.3" + p-map "^7.0.1" + package-config "^5.0.0" + picomatch "^3.0.1" + plur "^5.1.0" + pretty-ms "^9.0.0" + resolve-cwd "^3.0.0" + stack-utils "^2.0.6" + strip-ansi "^7.1.0" + supertap "^3.0.1" + temp-dir "^3.0.0" + write-file-atomic "^5.0.1" + yargs "^17.7.2" + +babel-eslint@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232" + integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg== + dependencies: + "@babel/code-frame" "^7.0.0" + "@babel/parser" "^7.7.0" + "@babel/traverse" "^7.7.0" + "@babel/types" "^7.7.0" + eslint-visitor-keys "^1.0.0" + resolve "^1.12.0" + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +bindings@^1.4.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + +blueimp-md5@^2.10.0: + version "2.19.0" + resolved "https://registry.yarnpkg.com/blueimp-md5/-/blueimp-md5-2.19.0.tgz#b53feea5498dcb53dc6ec4b823adb84b729c4af0" + integrity sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w== + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + +buffer-from@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== + +builtin-modules@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6" + integrity sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw== + +callsites@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== + +callsites@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-4.2.0.tgz#98761d5be3ce092e4b9c92f7fb8c8eb9b83cadc8" + integrity sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ== + +cbor@^9.0.1: + version "9.0.2" + resolved "https://registry.yarnpkg.com/cbor/-/cbor-9.0.2.tgz#536b4f2d544411e70ec2b19a2453f10f83cd9fdb" + integrity sha512-JPypkxsB10s9QOWwa6zwPzqE1Md3vqpPc+cai4sAecuCsRyAtAl/pMyhPlMbT/xtPnm2dznJZYRLui57qiRhaQ== + dependencies: + nofilter "^3.1.0" + +chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +chalk@^4.0.0, chalk@^4.1.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +chalk@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" + integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== + +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + +chunkd@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/chunkd/-/chunkd-2.0.1.tgz#49cd1d7b06992dc4f7fccd962fe2a101ee7da920" + integrity sha512-7d58XsFmOq0j6el67Ug9mHf9ELUXsQXYJBkyxhH/k+6Ke0qXRnv0kbemx+Twc6fRJ07C49lcbdgm9FL1Ei/6SQ== + +ci-info@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.0.0.tgz#65466f8b280fc019b9f50a5388115d17a63a44f2" + integrity sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg== + +ci-parallel-vars@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/ci-parallel-vars/-/ci-parallel-vars-1.0.1.tgz#e87ff0625ccf9d286985b29b4ada8485ca9ffbc2" + integrity sha512-uvzpYrpmidaoxvIQHM+rKSrigjOe9feHYbw4uOI2gdfe1C3xIlxO+kVXq83WQWNniTf8bAxVpy+cQeFQsMERKg== + +cli-truncate@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-4.0.0.tgz#6cc28a2924fee9e25ce91e973db56c7066e6172a" + integrity sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA== + dependencies: + slice-ansi "^5.0.0" + string-width "^7.0.0" + +cliui@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa" + integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ== + dependencies: + string-width "^4.2.0" + strip-ansi "^6.0.1" + wrap-ansi "^7.0.0" + +code-excerpt@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/code-excerpt/-/code-excerpt-4.0.0.tgz#2de7d46e98514385cb01f7b3b741320115f4c95e" + integrity sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA== + dependencies: + convert-to-spaces "^2.0.1" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +color-support@^1.1.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" + integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== + +commander@^2.20.0: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + +common-path-prefix@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-3.0.0.tgz#7d007a7e07c58c4b4d5f433131a19141b29f11e0" + integrity sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w== + +commondir@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" + integrity sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== + +concordance@^5.0.4: + version "5.0.4" + resolved "https://registry.yarnpkg.com/concordance/-/concordance-5.0.4.tgz#9896073261adced72f88d60e4d56f8efc4bbbbd2" + integrity sha512-OAcsnTEYu1ARJqWVGwf4zh4JDfHZEaSNlNccFmt8YjB2l/n19/PF2viLINHc57vO4FKIAFl2FWASIGZZWZ2Kxw== + dependencies: + date-time "^3.1.0" + esutils "^2.0.3" + fast-diff "^1.2.0" + js-string-escape "^1.0.1" + lodash "^4.17.15" + md5-hex "^3.0.1" + semver "^7.3.2" + well-known-symbols "^2.0.0" + +console-control-strings@^1.0.0, console-control-strings@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" + integrity sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ== + +convert-to-spaces@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz#61a6c98f8aa626c16b296b862a91412a33bceb6b" + integrity sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ== + +cross-spawn@^7.0.0, cross-spawn@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + +currently-unhandled@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" + integrity sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng== + dependencies: + array-find-index "^1.0.1" + +date-time@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/date-time/-/date-time-3.1.0.tgz#0d1e934d170579f481ed8df1e2b8ff70ee845e1e" + integrity sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg== + dependencies: + time-zone "^1.0.0" + +dateformat@^4.2.1: + version "4.6.3" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" + integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== + +debug@4, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + +deep-is@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" + integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== + +deepmerge@^4.2.2: + version "4.3.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== + +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== + +detect-libc@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700" + integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw== + +eastasianwidth@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== + +emittery@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/emittery/-/emittery-1.0.3.tgz#c9d2a9c689870f15251bb13b31c67715c26d69ac" + integrity sha512-tJdCJitoy2lrC2ldJcqN4vkqJ00lT+tOWNT1hBJjO/3FDMJa5TTIiYGCKGkn/WfCyOzUMObeohbVTj00fhiLiA== + +emoji-regex@^10.3.0: + version "10.4.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-10.4.0.tgz#03553afea80b3975749cfcb36f776ca268e413d4" + integrity sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw== + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +emoji-regex@^9.2.2: + version "9.2.2" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" + integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== + +escalade@^3.1.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" + integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== + +escape-string-regexp@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" + integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== + +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + +escape-string-regexp@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" + integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== + +eslint-config-prettier@9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz#31af3d94578645966c082fcb71a5846d3c94867f" + integrity sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw== + +eslint-scope@^8.0.2: + version "8.1.0" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-8.1.0.tgz#70214a174d4cbffbc3e8a26911d8bf51b9ae9d30" + integrity sha512-14dSvlhaVhKKsa9Fx1l8A17s7ah7Ef7wCakJ10LYk6+GYmP9yDti2oq2SEwcyndt6knfcZyhyxwY3i9yL78EQw== + dependencies: + esrecurse "^4.3.0" + estraverse "^5.2.0" + +eslint-visitor-keys@^1.0.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== + +eslint-visitor-keys@^3.3.0: + version "3.4.3" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" + integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== + +eslint-visitor-keys@^4.0.0, eslint-visitor-keys@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.1.0.tgz#1f785cc5e81eb7534523d85922248232077d2f8c" + integrity sha512-Q7lok0mqMUSf5a/AdAZkA5a/gHcO6snwQClVNNvFKCAVlxXucdU8pKydU5ZVZjBx5xr37vGbFFWtLQYreLzrZg== + +eslint@9.9.1: + version "9.9.1" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.9.1.tgz#147ac9305d56696fb84cf5bdecafd6517ddc77ec" + integrity sha512-dHvhrbfr4xFQ9/dq+jcVneZMyRYLjggWjk6RVsIiHsP8Rz6yZ8LvZ//iU4TrZF+SXWG+JkNF2OyiZRvzgRDqMg== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" + "@eslint-community/regexpp" "^4.11.0" + "@eslint/config-array" "^0.18.0" + "@eslint/eslintrc" "^3.1.0" + "@eslint/js" "9.9.1" + "@humanwhocodes/module-importer" "^1.0.1" + "@humanwhocodes/retry" "^0.3.0" + "@nodelib/fs.walk" "^1.2.8" + ajv "^6.12.4" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.3.2" + escape-string-regexp "^4.0.0" + eslint-scope "^8.0.2" + eslint-visitor-keys "^4.0.0" + espree "^10.1.0" + esquery "^1.5.0" + esutils "^2.0.2" + fast-deep-equal "^3.1.3" + file-entry-cache "^8.0.0" + find-up "^5.0.0" + glob-parent "^6.0.2" + ignore "^5.2.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + is-path-inside "^3.0.3" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash.merge "^4.6.2" + minimatch "^3.1.2" + natural-compare "^1.4.0" + optionator "^0.9.3" + strip-ansi "^6.0.1" + text-table "^0.2.0" + +espree@^10.0.1, espree@^10.1.0: + version "10.2.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-10.2.0.tgz#f4bcead9e05b0615c968e85f83816bc386a45df6" + integrity sha512-upbkBJbckcCNBDBDXEbuhjbP68n+scUd3k/U2EkyM9nw+I/jPiL4cLF/Al06CF96wRltFda16sxDFrxsI1v0/g== + dependencies: + acorn "^8.12.0" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^4.1.0" + +esprima@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +esquery@^1.5.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.6.0.tgz#91419234f804d852a82dceec3e16cdc22cf9dae7" + integrity sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg== + dependencies: + estraverse "^5.1.0" + +esrecurse@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== + dependencies: + estraverse "^5.2.0" + +estraverse@^5.1.0, estraverse@^5.2.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" + integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== + +estree-walker@2.0.2, estree-walker@^2.0.1, estree-walker@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-2.0.2.tgz#52f010178c2a4c117a7757cfe942adb7d2da4cac" + integrity sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w== + +esutils@^2.0.2, esutils@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-diff@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.3.0.tgz#ece407fa550a64d638536cd727e129c61616e0f0" + integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== + +fast-glob@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" + integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +fast-levenshtein@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== + +fastq@^1.6.0: + version "1.17.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47" + integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w== + dependencies: + reusify "^1.0.4" + +figures@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-6.1.0.tgz#935479f51865fa7479f6fa94fc6fc7ac14e62c4a" + integrity sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg== + dependencies: + is-unicode-supported "^2.0.0" + +file-entry-cache@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f" + integrity sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ== + dependencies: + flat-cache "^4.0.0" + +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + +find-up-simple@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/find-up-simple/-/find-up-simple-1.0.0.tgz#21d035fde9fdbd56c8f4d2f63f32fd93a1cfc368" + integrity sha512-q7Us7kcjj2VMePAa02hDAF6d+MzsdsAWEwYyOpwUtlerRBkOEPBCRZrAV4XfcSN8fHAgaD0hP7miwoay6DCprw== + +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + +flat-cache@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-4.0.1.tgz#0ece39fcb14ee012f4b0410bd33dd9c1f011127c" + integrity sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw== + dependencies: + flatted "^3.2.9" + keyv "^4.5.4" + +flatted@^3.2.9: + version "3.3.1" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" + integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== + +foreground-child@^3.1.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.0.tgz#0ac8644c06e431439f8561db8ecf29a7b5519c77" + integrity sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg== + dependencies: + cross-spawn "^7.0.0" + signal-exit "^4.0.1" + +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== + +fsevents@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + +gauge@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-3.0.2.tgz#03bf4441c044383908bcfa0656ad91803259b395" + integrity sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q== + dependencies: + aproba "^1.0.3 || ^2.0.0" + color-support "^1.1.2" + console-control-strings "^1.0.0" + has-unicode "^2.0.1" + object-assign "^4.1.1" + signal-exit "^3.0.0" + string-width "^4.2.3" + strip-ansi "^6.0.1" + wide-align "^1.1.2" + +get-caller-file@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + +get-east-asian-width@^1.0.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz#21b4071ee58ed04ee0db653371b55b4299875389" + integrity sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ== + +glob-parent@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== + dependencies: + is-glob "^4.0.1" + +glob-parent@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" + integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== + dependencies: + is-glob "^4.0.3" + +glob@^10.4.1: + version "10.4.5" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" + integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^1.11.1" + +glob@^7.1.3: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + +globals@^14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" + integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== + +globby@^14.0.0: + version "14.0.2" + resolved "https://registry.yarnpkg.com/globby/-/globby-14.0.2.tgz#06554a54ccfe9264e5a9ff8eded46aa1e306482f" + integrity sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw== + dependencies: + "@sindresorhus/merge-streams" "^2.1.0" + fast-glob "^3.3.2" + ignore "^5.2.4" + path-type "^5.0.0" + slash "^5.1.0" + unicorn-magic "^0.1.0" + +graceful-fs@^4.2.9: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-unicode@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" + integrity sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ== + +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + +https-proxy-agent@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" + integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== + dependencies: + agent-base "6" + debug "4" + +ignore-by-default@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-2.1.0.tgz#c0e0de1a99b6065bdc93315a6f728867981464db" + integrity sha512-yiWd4GVmJp0Q6ghmM2B/V3oZGRmjrKLXvHR3TE1nfoXsmoggllfZUQe74EN0fJdPFZu2NIvNdrMMLm3OsV7Ohw== + +ignore@^5.2.0, ignore@^5.2.4: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" + integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== + +import-fresh@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== + dependencies: + parent-module "^1.0.0" + resolve-from "^4.0.0" + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== + +indent-string@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5" + integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg== + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@^2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +irregular-plurals@^3.3.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/irregular-plurals/-/irregular-plurals-3.5.0.tgz#0835e6639aa8425bdc8b0d33d0dc4e89d9c01d2b" + integrity sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ== + +is-builtin-module@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" + integrity sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A== + dependencies: + builtin-modules "^3.3.0" + +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + +is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-fullwidth-code-point@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz#fae3167c729e7463f8461ce512b080a49268aa88" + integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ== + +is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== + dependencies: + is-extglob "^2.1.1" + +is-module@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" + integrity sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g== + +is-number@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== + +is-path-inside@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== + +is-plain-object@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" + integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== + +is-promise@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-4.0.0.tgz#42ff9f84206c1991d26debf520dd5c01042dd2f3" + integrity sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ== + +is-reference@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7" + integrity sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ== + dependencies: + "@types/estree" "*" + +is-unicode-supported@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz#09f0ab0de6d3744d48d265ebb98f65d11f2a9b3a" + integrity sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ== + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + +jackspeak@^3.1.2: + version "3.4.3" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.3.tgz#8833a9d89ab4acde6188942bd1c53b6390ed5a8a" + integrity sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + +js-string-escape@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" + integrity sha512-Smw4xcfIQ5LVjAOuJCvN/zIodzA/BBSsluuoSykP+lUvScIi4U6RJLfwHet5cxFnCswUjISV8oAXaqaJDY3chg== + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +js-yaml@^3.14.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + +jsesc@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e" + integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== + +json-buffer@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" + integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-stable-stringify-without-jsonify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== + +keyv@^4.5.4: + version "4.5.4" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== + dependencies: + json-buffer "3.0.1" + +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== + dependencies: + prelude-ls "^1.2.1" + type-check "~0.4.0" + +load-json-file@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-7.0.1.tgz#a3c9fde6beffb6bedb5acf104fad6bb1604e1b00" + integrity sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ== + +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + +lodash.merge@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== + +lodash@^4.17.15, lodash@^4.17.20: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + +lru-cache@^10.2.0: + version "10.4.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" + integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== + +magic-string@^0.30.3: + version "0.30.12" + resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.12.tgz#9eb11c9d072b9bcb4940a5b2c2e1a217e4ee1a60" + integrity sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw== + dependencies: + "@jridgewell/sourcemap-codec" "^1.5.0" + +make-dir@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" + integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== + dependencies: + semver "^6.0.0" + +matcher@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/matcher/-/matcher-5.0.0.tgz#cd82f1c7ae7ee472a9eeaf8ec7cac45e0fe0da62" + integrity sha512-s2EMBOWtXFc8dgqvoAzKJXxNHibcdJMV0gwqKUaw9E2JBJuGUK7DrNKrA6g/i+v72TT16+6sVm5mS3thaMLQUw== + dependencies: + escape-string-regexp "^5.0.0" + +md5-hex@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/md5-hex/-/md5-hex-3.0.1.tgz#be3741b510591434b2784d79e556eefc2c9a8e5c" + integrity sha512-BUiRtTtV39LIJwinWBjqVsU9xhdnz7/i889V859IBFpuqGAj6LuOvHv5XLbgZ2R7ptJoJaEcxkv88/h25T7Ciw== + dependencies: + blueimp-md5 "^2.10.0" + +memoize@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/memoize/-/memoize-10.0.0.tgz#43fa66b2022363c7c50cf5dfab732a808a3d7147" + integrity sha512-H6cBLgsi6vMWOcCpvVCdFFnl3kerEXbrYh9q+lY6VXvQSmM6CkmV08VOwT+WE2tzIEqRPFfAq3fm4v/UIW6mSA== + dependencies: + mimic-function "^5.0.0" + +merge2@^1.3.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" + integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== + +micromatch@^4.0.2, micromatch@^4.0.4: + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== + dependencies: + braces "^3.0.3" + picomatch "^2.3.1" + +mimic-function@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/mimic-function/-/mimic-function-5.0.1.tgz#acbe2b3349f99b9deaca7fb70e48b83e94e67076" + integrity sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA== + +minimatch@^3.1.1, minimatch@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimatch@^9.0.4: + version "9.0.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" + integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow== + dependencies: + brace-expansion "^2.0.1" + +minipass@^3.0.0: + version "3.3.6" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" + integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== + dependencies: + yallist "^4.0.0" + +minipass@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" + integrity sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ== + +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + +minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +natural-compare@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== + +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + +node-gyp-build@^4.2.2: + version "4.8.2" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.2.tgz#4f802b71c1ab2ca16af830e6c1ea7dd1ad9496fa" + integrity sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw== + +nofilter@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/nofilter/-/nofilter-3.1.0.tgz#c757ba68801d41ff930ba2ec55bab52ca184aa66" + integrity sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g== + +nopt@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" + integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ== + dependencies: + abbrev "1" + +npmlog@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0" + integrity sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw== + dependencies: + are-we-there-yet "^2.0.0" + console-control-strings "^1.1.0" + gauge "^3.0.0" + set-blocking "^2.0.0" + +object-assign@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + +optionator@^0.9.3: + version "0.9.4" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" + integrity sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g== + dependencies: + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + word-wrap "^1.2.5" + +p-limit@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + +p-map@^7.0.1: + version "7.0.2" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-7.0.2.tgz#7c5119fada4755660f70199a66aa3fe2f85a1fe8" + integrity sha512-z4cYYMMdKHzw4O5UkWJImbZynVIo0lSGTXc7bzB1e/rrDqkgGUNysK/o4bTr+0+xKvvLoTyGqYC4Fgljy9qe1Q== + +package-config@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/package-config/-/package-config-5.0.0.tgz#cba78b7feb3396fa0149caca2c72677ff302b3c4" + integrity sha512-GYTTew2slBcYdvRHqjhwaaydVMvn/qrGC323+nKclYioNSLTDUM/lGgtGTgyHVtYcozb+XkE8CNhwcraOmZ9Mg== + dependencies: + find-up-simple "^1.0.0" + load-json-file "^7.0.1" + +package-json-from-dist@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz#4f1471a010827a86f94cfd9b0727e36d267de505" + integrity sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw== + +parent-module@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== + dependencies: + callsites "^3.0.0" + +parse-ms@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/parse-ms/-/parse-ms-4.0.0.tgz#c0c058edd47c2a590151a718990533fd62803df4" + integrity sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw== + +path-exists@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== + +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + +path-type@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-5.0.0.tgz#14b01ed7aea7ddf9c7c3f46181d4d04f9c785bb8" + integrity sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg== + +picocolors@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + +picomatch@^2.2.2, picomatch@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + +picomatch@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-3.0.1.tgz#817033161def55ec9638567a2f3bbc876b3e7516" + integrity sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag== + +plur@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/plur/-/plur-5.1.0.tgz#bff58c9f557b9061d60d8ebf93959cf4b08594ae" + integrity sha512-VP/72JeXqak2KiOzjgKtQen5y3IZHn+9GOuLDafPv0eXa47xq0At93XahYBs26MsifCQ4enGKwbjBTKgb9QJXg== + dependencies: + irregular-plurals "^3.3.0" + +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== + +prettier@3.3.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.3.3.tgz#30c54fe0be0d8d12e6ae61dbb10109ea00d53105" + integrity sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew== + +pretty-ms@^9.0.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/pretty-ms/-/pretty-ms-9.1.0.tgz#0ad44de6086454f48a168e5abb3c26f8db1b3253" + integrity sha512-o1piW0n3tgKIKCwk2vpM/vOV13zjJzvP37Ioze54YlTHE06m4tjEbzg9WsKkvTuyYln2DHjo5pY4qrZGI0otpw== + dependencies: + parse-ms "^4.0.0" + +punycode@^2.1.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + +queue-microtask@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" + integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== + +randombytes@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" + integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ== + dependencies: + safe-buffer "^5.1.0" + +readable-stream@^3.6.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== + +resolve-cwd@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" + integrity sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg== + dependencies: + resolve-from "^5.0.0" + +resolve-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== + +resolve-from@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" + integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== + +resolve@^1.12.0, resolve@^1.22.1: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + +reusify@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== + +rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + +rollup-plugin-version-injector@^1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/rollup-plugin-version-injector/-/rollup-plugin-version-injector-1.3.3.tgz#df1f8d371bc127592c29aeda6914745db4013a48" + integrity sha512-+Rrf0xIFHkwFGuMfphVlAOtd9FlhHFh3vrDwamJ6+YR3IxebRHGVT879qwWzZ1CpWMCLlngb2MmHW5wC5EJqvg== + dependencies: + chalk "^4.1.0" + dateformat "^4.2.1" + lodash "^4.17.20" + +rollup@4.21.2: + version "4.21.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.21.2.tgz#f41f277a448d6264e923dd1ea179f0a926aaf9b7" + integrity sha512-e3TapAgYf9xjdLvKQCkQTnbTKd4a6jwlpQSJJFokHGaX2IVjoEqkIIhiQfqsi0cdwlOD+tQGuOd5AJkc5RngBw== + dependencies: + "@types/estree" "1.0.5" + optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.21.2" + "@rollup/rollup-android-arm64" "4.21.2" + "@rollup/rollup-darwin-arm64" "4.21.2" + "@rollup/rollup-darwin-x64" "4.21.2" + "@rollup/rollup-linux-arm-gnueabihf" "4.21.2" + "@rollup/rollup-linux-arm-musleabihf" "4.21.2" + "@rollup/rollup-linux-arm64-gnu" "4.21.2" + "@rollup/rollup-linux-arm64-musl" "4.21.2" + "@rollup/rollup-linux-powerpc64le-gnu" "4.21.2" + "@rollup/rollup-linux-riscv64-gnu" "4.21.2" + "@rollup/rollup-linux-s390x-gnu" "4.21.2" + "@rollup/rollup-linux-x64-gnu" "4.21.2" + "@rollup/rollup-linux-x64-musl" "4.21.2" + "@rollup/rollup-win32-arm64-msvc" "4.21.2" + "@rollup/rollup-win32-ia32-msvc" "4.21.2" + "@rollup/rollup-win32-x64-msvc" "4.21.2" + fsevents "~2.3.2" + +run-parallel@^1.1.9: + version "1.2.0" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" + integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== + dependencies: + queue-microtask "^1.2.2" + +safe-buffer@^5.1.0, safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +semver@^6.0.0: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.3.2, semver@^7.3.5: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +serialize-error@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18" + integrity sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw== + dependencies: + type-fest "^0.13.1" + +serialize-javascript@^6.0.1: + version "6.0.2" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz#defa1e055c83bf6d59ea805d8da862254eb6a6c2" + integrity sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g== + dependencies: + randombytes "^2.1.0" + +set-blocking@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== + +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +signal-exit@^3.0.0: + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== + +signal-exit@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" + integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== + +slash@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-5.1.0.tgz#be3adddcdf09ac38eebe8dcdc7b1a57a75b095ce" + integrity sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg== + +slice-ansi@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-5.0.0.tgz#b73063c57aa96f9cd881654b15294d95d285c42a" + integrity sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ== + dependencies: + ansi-styles "^6.0.0" + is-fullwidth-code-point "^4.0.0" + +smob@^1.0.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/smob/-/smob-1.5.0.tgz#85d79a1403abf128d24d3ebc1cdc5e1a9548d3ab" + integrity sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig== + +source-map-support@~0.5.20: + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map@^0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== + +stack-utils@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" + integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== + dependencies: + escape-string-regexp "^2.0.0" + +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^5.0.1, string-width@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== + dependencies: + eastasianwidth "^0.2.0" + emoji-regex "^9.2.2" + strip-ansi "^7.0.1" + +string-width@^7.0.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-7.2.0.tgz#b5bb8e2165ce275d4d43476dd2700ad9091db6dc" + integrity sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ== + dependencies: + emoji-regex "^10.3.0" + get-east-asian-width "^1.0.0" + strip-ansi "^7.1.0" + +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^7.0.1, strip-ansi@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== + dependencies: + ansi-regex "^6.0.1" + +strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + +supertap@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/supertap/-/supertap-3.0.1.tgz#aa89e4522104402c6e8fe470a7d2db6dc4037c6a" + integrity sha512-u1ZpIBCawJnO+0QePsEiOknOfCRq0yERxiAchT0i4li0WHNUJbf0evXXSXOcCAR4M8iMDoajXYmstm/qO81Isw== + dependencies: + indent-string "^5.0.0" + js-yaml "^3.14.1" + serialize-error "^7.0.1" + strip-ansi "^7.0.1" + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +tar@^6.1.11: + version "6.2.1" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" + integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^5.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + +temp-dir@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-3.0.0.tgz#7f147b42ee41234cc6ba3138cd8e8aa2302acffa" + integrity sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw== + +terser@^5.17.4: + version "5.36.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.36.0.tgz#8b0dbed459ac40ff7b4c9fd5a3a2029de105180e" + integrity sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w== + dependencies: + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" + commander "^2.20.0" + source-map-support "~0.5.20" + +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== + +time-zone@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/time-zone/-/time-zone-1.0.0.tgz#99c5bf55958966af6d06d83bdf3800dc82faec5d" + integrity sha512-TIsDdtKo6+XrPtiTm1ssmMngN1sAhyKnTO2kunQWqNPWIVvCm15Wmw4SWInwTVgJ5u/Tr04+8Ei9TNcw4x4ONA== + +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== + +to-regex-range@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== + dependencies: + is-number "^7.0.0" + +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== + dependencies: + prelude-ls "^1.2.1" + +type-fest@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" + integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== + +unicorn-magic@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/unicorn-magic/-/unicorn-magic-0.1.0.tgz#1bb9a51c823aaf9d73a8bfcd3d1a23dde94b0ce4" + integrity sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ== + +uri-js@^4.2.2, uri-js@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + dependencies: + punycode "^2.1.0" + +util-deprecate@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== + +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +well-known-symbols@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/well-known-symbols/-/well-known-symbols-2.0.0.tgz#e9c7c07dbd132b7b84212c8174391ec1f9871ba5" + integrity sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +wide-align@^1.1.2: + version "1.1.5" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3" + integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg== + dependencies: + string-width "^1.0.2 || 2 || 3 || 4" + +word-wrap@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" + integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== + +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== + dependencies: + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== + +write-file-atomic@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-5.0.1.tgz#68df4717c55c6fa4281a7860b4c2ba0a6d2b11e7" + integrity sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw== + dependencies: + imurmurhash "^0.1.4" + signal-exit "^4.0.1" + +y18n@^5.0.5: + version "5.0.8" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" + integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + +yargs-parser@^21.1.1: + version "21.1.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" + integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== + +yargs@^17.7.2: + version "17.7.2" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" + integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== + dependencies: + cliui "^8.0.1" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.3" + y18n "^5.0.5" + yargs-parser "^21.1.1" + +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== diff --git a/openzim.toml b/openzim.toml new file mode 100644 index 00000000..8db2dc8e --- /dev/null +++ b/openzim.toml @@ -0,0 +1,15 @@ +[files.assets.config] +target_dir="src/zimscraperlib/rewriting/statics" +execute_after=[ + "cd ../../../../ && python rules/generate_rules.py", # generate Python (and JS) rules +] + +[files.assets.actions."wombat.js"] +action="get_file" +source="https://cdn.jsdelivr.net/npm/@webrecorder/wombat@3.8.2/dist/wombat.js" +target_file="wombat.js" + +[files.assets.actions."wombatSetup.js"] # fallback if this script has not been properly built +action="get_file" +source="https://dev.kiwix.org/zimscraperlib/wombatSetup.js" +target_file="wombatSetup.js" diff --git a/pyproject.toml b/pyproject.toml index d4528c1c..c0149861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,12 @@ [build-system] -requires = ["hatchling", "hatch-openzim>=0.2"] +# jinja2 is required to generate JS and Python rules at build time +# PyYAML is used to parse fuzzy rules and generate Python/JS code +requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.4", "PyYAML==6.0.2"] build-backend = "hatchling.build" [project] name = "zimscraperlib" -requires-python = ">=3.8,<3.13" +requires-python = ">=3.12,<3.13" description = "Collection of python tools to re-use common code across scrapers" readme = "README.md" dependencies = [ @@ -36,12 +38,20 @@ additional-classifiers = [ "Intended Audience :: Developers", ] +[tool.hatch.build.hooks.openzim-build] + [project.optional-dependencies] scripts = [ "invoke==2.2.0", + # jinja2 is required to generate JS and Python rules at build time + # PyYAML is used to parse fuzzy rules and generate Python/JS code + # also update version in build-system above and in build_js.sh + "jinja2==3.1.4", + "PyYAML==6.0.2", + ] lint = [ - "black==24.4.2", + "black==24.10.0", "ruff==0.4.9", ] check = [ @@ -75,6 +85,11 @@ exclude = [ [tool.hatch.build.targets.wheel] packages = ["src/zimscraperlib"] +artifacts = [ + "src/zimscraperlib/rewriting/statics/**", + "src/zimscraperlib/rewriting/rules.py", + "tests/rewriting/test_fuzzy_rules.py", +] [tool.hatch.envs.default] features = ["dev"] @@ -82,9 +97,6 @@ features = ["dev"] [tool.hatch.envs.test] features = ["scripts", "test"] -[[tool.hatch.envs.test.matrix]] -python = ["3.8", "3.9", "3.10", "3.11", "3.12"] - [tool.hatch.envs.test.scripts] run = "inv test --args '{args}'" run-cov = "inv test-cov --args '{args}'" @@ -114,10 +126,10 @@ all = "inv checkall --args '{args}'" [tool.black] line-length = 88 -target-version = ['py38'] +target-version = ['py312'] [tool.ruff] -target-version = "py38" +target-version = "py312" line-length = 88 src = ["src", "contrib"] @@ -244,6 +256,15 @@ exclude_lines = [ include = ["contrib", "src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**"] extraPaths = ["src"] -pythonVersion = "3.8" +pythonVersion = "3.12" typeCheckingMode="basic" disableBytesTypePromotions = true + +[tool.pyright.overrides] +strict = true # Enable strict mode for specific files + +[[tool.pyright.overrides.files]] +files = [ + "src/zimscraperlib/rewriting**/*.py", + "tests/rewriting/**/*.py" +] diff --git a/rules/generate_rules.py b/rules/generate_rules.py new file mode 100644 index 00000000..cd9ca8e5 --- /dev/null +++ b/rules/generate_rules.py @@ -0,0 +1,180 @@ +import logging +import re +from pathlib import Path + +import yaml +from jinja2 import Environment + +REQUEST_TIMEOUT = 10 + +logging.basicConfig(level=logging.INFO) + +rules_src = Path(__file__).with_name("rules.yaml") +if not rules_src.exists(): + logging.error("Skipping rules generation, rule file is missing") + raise SystemExit(1) + +rules_config = yaml.safe_load(rules_src.read_text()) + +FUZZY_RULES = rules_config["fuzzyRules"] + +for rule in FUZZY_RULES: + if "name" not in rule: + raise SystemExit("Fuzzy rule is missing a name") + if "tests" not in rule or len(rule["tests"]) == 0: + raise SystemExit("Fuzzy rule is missing test cases") + + +PY2JS_RULE_RX = re.compile(r"\\(\d)", re.ASCII) + +# Do not escape anything, we want to generate code as-is, it won't be interpreted as +# HTML anyway +JINJA_ENV = Environment(autoescape=False) # noqa: S701 # nosec: B701 + +### Generate Javascript code + +js_code_template = """// THIS IS AN AUTOMATICALLY GENERATED FILE, DO NOT MODIFY DIRECTLY + +export const fuzzyRules = [ +{% for rule in FUZZY_RULES %} { + match: '{{ rule['match'] }}', + replace: '{{ rule['replace'] }}', + }, +{% endfor %} +]; + +""" + +js_parent = Path(__file__).joinpath("../../javascript/src").resolve() +if not js_parent.exists(): + # This skip is usefull mostly for CI operations when working on the Python part + logging.warning("Skipping JS rules generation, target folder is missing") +else: + (js_parent / "fuzzyRules.js").write_text( + JINJA_ENV.from_string(js_code_template).render( + FUZZY_RULES=[ + { + "match": rule["pattern"].replace("\\", "\\\\"), + "replace": PY2JS_RULE_RX.sub(r"$\1", rule["replace"]), + } + for rule in FUZZY_RULES + ] + ) + ) + logging.info("JS rules generation completed successfully") + +### Generate Javascript tests + +js_test_template = """// THIS IS AN AUTOMATICALLY GENERATED FILE, DO NOT MODIFY DIRECTLY + +import test from 'ava'; + +import { applyFuzzyRules } from '../src/wombatSetup.js'; + +{% for rule in FUZZY_RULES %} +{% for test in rule['tests'] %} +test('fuzzyrules_{{rule['name']}}_{{loop.index}}', (t) => { + t.is( + applyFuzzyRules( + '{{test['raw_url']}}', + ), + '{{test['raw_url'] if test['unchanged'] else test['fuzzified_url']}}', + ); +}); +{% endfor %} +{% endfor %} +""" + +js_parent = Path(__file__).joinpath("../../javascript/test").resolve() +if not js_parent.exists(): + # This skip is usefull mostly for CI operations when working on the Python part + logging.warning("Skipping JS tests generation, target folder is missing") +else: + (js_parent / "fuzzyRules.js").write_text( + JINJA_ENV.from_string(js_test_template).render( + FUZZY_RULES=[ + { + "name": rule["name"], + "tests": rule["tests"], + "match": rule["pattern"].replace("\\", "\\\\"), + "replace": PY2JS_RULE_RX.sub(r"$\1", rule["replace"]), + } + for rule in FUZZY_RULES + ] + ) + ) + logging.info("JS tests generation completed successfully") + +### Generate Python code + +py_code_template = """# THIS IS AN AUTOMATICALLY GENERATED FILE, DO NOT MODIFY DIRECTLY + +FUZZY_RULES = [ +{% for rule in FUZZY_RULES %} { + "pattern": r"{{ rule['pattern'] }}", + "replace": r"{{ rule['replace'] }}", + }, +{% endfor %} +] + +""" + +py_parent = Path(__file__).joinpath("../../src/zimscraperlib/rewriting").resolve() +if not py_parent.exists(): + # This skip is usefull mostly for CI operations when working on the JS part + logging.warning("Skipping Python rules generation, target folder is missing") +else: + (py_parent / "rules.py").absolute().write_text( + JINJA_ENV.from_string(py_code_template).render(FUZZY_RULES=FUZZY_RULES) + ) + logging.info("Python rules generation completed successfully") + +### Generate Python tests + +py_test_template = """# THIS IS AN AUTOMATICALLY GENERATED FILE, DO NOT MODIFY DIRECTLY + +import pytest + +from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter + +from .utils import ContentForTests + +{% for rule in FUZZY_RULES %} +@pytest.fixture( + params=[ +{% for test in rule['tests'] %} +{% if test['unchanged'] %} + ContentForTests( + "{{ test['raw_url'] }}", + ), +{% else %} + ContentForTests( + "{{ test['raw_url'] }}", + "{{ test['fuzzified_url'] }}", + ), +{% endif %} +{% endfor %} + ] +) +def {{ rule['name'] }}_case(request): + yield request.param + + +def test_fuzzyrules_{{ rule['name'] }}({{ rule['name'] }}_case): + assert ( + ArticleUrlRewriter.apply_additional_rules({{ rule['name'] }}_case.input_str) + == {{ rule['name'] }}_case.expected_str + ) +{% endfor %} + +""" + +py_parent = Path(__file__).joinpath("../../tests/rewriting").resolve() +if not py_parent.exists(): + # This skip is usefull mostly for CI operations when working on the JS part + logging.warning("Skipping Python tests generation, target folder is missing") +else: + (py_parent / "test_fuzzy_rules.py").absolute().write_text( + JINJA_ENV.from_string(py_test_template).render(FUZZY_RULES=FUZZY_RULES) + ) + logging.info("Python tests generation completed successfully") diff --git a/rules/rules.yaml b/rules/rules.yaml new file mode 100644 index 00000000..bc5206eb --- /dev/null +++ b/rules/rules.yaml @@ -0,0 +1,213 @@ +# This file comes from an adaptation of rules present in +# https://github.com/webrecorder/wabac.js/blame/main/src/fuzzymatcher.js +# +# Syncing rules is done manually, based on expert knowledge, especially because in +# scraperlib we are not really fuzzy matching (searching the best entry among existing +# ones) but just rewriting to proper path. +# +# This file is in sync with content at commit 879018d5b96962df82340a9a57570bbc0fc67815 +# from June 9, 2024 +# +# This file should be updated at every release of scraperlib +# +# Some rules are voluntarily missing because not been tested in scraperlib yet: Twitter, +# Washington Post, WixStatic, Facebook +# +# Generic rules are also ommitted on purpose, we don't need them +# +fuzzyRules: + - name: googlevideo_com + pattern: .*googlevideo.com/(videoplayback(?=\?)).*[?&](id=[^&]+).* + replace: youtube.fuzzy.replayweb.page/\1?\2 + tests: + - raw_url: foobargooglevideo.com/videoplayback?id=1576&key=value + fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576 + - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576 + fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576 + - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576&key=value + fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576 + - raw_url: foobargooglevideo.com/videoplaybackandfoo?some=thing&id=1576&key=value + unchanged: true # videoplayback is not followed by `?` + - raw_url: foobargoogle_video.com/videoplaybackandfoo?some=thing&id=1576&key=value + unchanged: true # No googlevideo.com in url + - name: youtube_video_info + pattern: (?:www\.)?youtube(?:-nocookie)?\.com/(get_video_info\?).*(video_id=[^&]+).* + replace : youtube.fuzzy.replayweb.page/\1\2 + tests: + - raw_url: www.youtube.com/get_video_info?video_id=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: www.youtube.com/get_video_info?foo=bar&video_id=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: www.youtube.com/get_video_info?video_id=123ah&foo=bar + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: youtube.com/get_video_info?video_id=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: youtube-nocookie.com/get_video_info?video_id=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: www.youtube-nocookie.com/get_video_info?video_id=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah + - raw_url: www.youtube-nocookie.com/get_video_info?foo=bar + unchanged: true # no video_id parameter + - raw_url: www.youtubeqnocookie.com/get_video_info?video_id=123ah + unchanged: true # improper hostname + - name: youtube_thumbnails + pattern: i\.ytimg\.com\/vi\/(.*?)\/.*?\.(\w*?)(?:\?.*|$) + replace : i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2 + tests: + - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g + fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg + - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.png?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g + fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.png + - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg + fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg + - raw_url: i.ytimg.com/vi/-KpLmsAR23I/max-res.default.jpg + fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg + - name: trim_digits_only + pattern: ([^?]+)\?[\d]+$ + replace : \1 + tests: + - raw_url: www.example.com/page?1234 + fuzzified_url: www.example.com/page + - raw_url: www.example.com/page?foo=1234 + unchanged: true + - raw_url: www.example.com/page1234 + unchanged: true + - raw_url: www.example.com/page?foo=bar&1234 + unchanged: true + - raw_url: www.example.com/page?1234=bar + unchanged: true + - raw_url: www.example.com/page?1234&foo=bar + unchanged: true + - name: youtubei + pattern: (?:www\.)?youtube(?:-nocookie)?\.com\/(youtubei\/[^?]+).*(videoId[^&]+).* + replace : youtube.fuzzy.replayweb.page/\1?\2 + tests: + - raw_url: www.youtube-nocookie.com/youtubei/page/?videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube-nocookie.com/youtubei/page/?videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube.com/youtubei/page/?videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: www.youtube.com/youtubei/page/?videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube.com/youtubei/page/videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube.com/youtubei/page/videoIdqqq=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoIdqqq=123ah + - raw_url: youtube.com/youtubei/page/videoId=123ah&foo=bar + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube.com/youtubei/page/?foo=bar&videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah + - raw_url: youtube.com/youtubei/page/foo=bar&videoId=123ah + fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/foo=bar&?videoId=123ah + - raw_url: youtube.com/youtubei/?videoId=123ah + unchanged: true + - name: youtube_embed + pattern: (?:www\.)?youtube(?:-nocookie)?\.com/embed/([^?]+).* + replace : youtube.fuzzy.replayweb.page/embed/\1 + tests: + - raw_url: www.youtube-nocookie.com/embed/foo + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo + - raw_url: www.youtube-nocookie.com/embed/bar + fuzzified_url: youtube.fuzzy.replayweb.page/embed/bar + - raw_url: www.youtube-nocookie.com/embed/foo/bar + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo/bar + - raw_url: www.youtube.com/embed/foo + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo + - raw_url: youtube.com/embed/foo + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo + - raw_url: youtube-nocookie.com/embed/foo + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo + - raw_url: youtube.com/embed/foo?bar=alice + fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo + + - name: vimeo_cdn_fix # custom warc2zim rule intended to fix Vimeo support + pattern: .*(?:gcs-vimeo|vod|vod-progressive|vod-adaptive)\.akamaized\.net.*\/(.+?.mp4)\?.*range=(.*?)(?:&.*|$) + replace : vimeo-cdn.fuzzy.replayweb.page/\1?range=\2 + tests: + - raw_url: gcs-vimeo.akamaized.net/123.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod.akamaized.net/123.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod-progressive.akamaized.net/123.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod-adaptive.akamaized.net/123.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod.akamaized.net/123.mp4?foo=bar&range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod.akamaized.net/123.mp4?foo=bar&range=123-456&bar=foo + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod.akamaized.net/123.mp4?range=123-456&bar=foo + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: foovod.akamaized.net/123.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 + - raw_url: vod.akamaized.net/1/23.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456 + - raw_url: vod.akamaized.net/a/23.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456 + - raw_url: vod.akamaized.net/foo/bar/23.mp4?range=123-456 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456 + - raw_url: foo.akamaized.net/123.mp4?range=123-456 + unchanged: true + - name: vimeo_cdn + pattern: .*(?:gcs-vimeo|vod|vod-progressive)\.akamaized\.net.*?\/([\d/]+.mp4)$ + replace : vimeo-cdn.fuzzy.replayweb.page/\1 + tests: + - raw_url: vod.akamaized.net/23.mp4 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4 + - raw_url: vod.akamaized.net/23/12332.mp4 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23/12332.mp4 + - raw_url: https://vod-progressive.akamaized.net/exp=1635528595~acl=%2Fvimeo-prod-skyfire-std-us%2F01%2F4423%2F13%2F347119375%2F1398505169.mp4~hmac=27c31f1990aab5e5429f7f7db5b2dcbcf8d2f5c92184d53102da36920d33d53e/vimeo-prod-skyfire-std-us/01/4423/13/347119375/1398505169.mp4 + fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/01/4423/13/347119375/1398505169.mp4 + - name: vimeo_player + pattern: .*player.vimeo.com\/(video\/[\d]+)\?.* + replace : vimeo.fuzzy.replayweb.page/\1 + tests: + - raw_url: player.vimeo.com/video/1234?foo=bar + fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234 + - raw_url: foo.player.vimeo.com/video/1234?foo=bar + fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234 + - raw_url: player.vimeo.com/video/1234?foo + fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234 + - raw_url: player.vimeo.com/video/1/23?foo=bar + unchanged: true + - raw_url: player.vimeo.com/video/123a?foo=bar + unchanged: true + - raw_url: player.vimeo.com/video/?foo=bar + unchanged: true + - name: i_vimeo_cdn + pattern: .*i\.vimeocdn\.com\/(.*)\?.* + replace : i.vimeocdn.fuzzy.replayweb.page/\1 + tests: + - raw_url: i.vimeocdn.com/image/1234?foo=bar + fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/image/1234 + - raw_url: i.vimeocdn.com/something/a456?foo + fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/something/a456 + - name: cheatography_com + pattern: cheatography\.com\/scripts\/(.*).js.*[?&](v=[^&]+).* + replace : cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2 + tests: + - raw_url: cheatography.com/scripts/useful.min.js?v=2&q=1719438924 + fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2 + - raw_url: cheatography.com/scripts/foo.js?v=2&q=1719438924 + fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/foo.js?v=2 + - raw_url: cheatography.com/scripts/useful.min.js?q=1719438924&v=2 + fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2 + - raw_url: cheatography.com/scripts/useful.min.js?q=1719438924&v=2&foo=bar + fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2 + - name: der_postillon_com + pattern: blogger.googleusercontent.com\/img\/(.*\.jpg)=.* + replace: blogger.googleusercontent.com.fuzzy.replayweb.page/img/\1.resized + tests: + - raw_url: blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/s600-rw/tickerzugtier2.jpg=w487-h220-p-k-no-nu + fuzzified_url: blogger.googleusercontent.com.fuzzy.replayweb.page/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/s600-rw/tickerzugtier2.jpg.resized + - raw_url: blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/w72-h72-p-k-no-nu/tickerzugtier2.jpg + unchanged: true + - name: iranwire_com + pattern: (iranwire\.com\/questions\/detail\/.*)\?.* + replace: \1 + tests: + - raw_url: iranwire.com/questions/detail/1723?&_=1721804954220 + fuzzified_url: iranwire.com/questions/detail/1723 + - raw_url: iranwire.com/questions/detail/1725?foo=bar&_=1721804454220 + fuzzified_url: iranwire.com/questions/detail/1725 diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index 865281cb..0e7ceb3c 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -29,7 +29,7 @@ import os import pathlib import subprocess -from typing import Callable +from collections.abc import Callable import piexif from optimize_images.img_aux_processing import do_reduce_colors, rebuild_palette diff --git a/src/zimscraperlib/rewriting/__init__.py b/src/zimscraperlib/rewriting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/zimscraperlib/rewriting/css.py b/src/zimscraperlib/rewriting/css.py new file mode 100644 index 00000000..d992ad9b --- /dev/null +++ b/src/zimscraperlib/rewriting/css.py @@ -0,0 +1,221 @@ +""" CSS Rewriting + +This modules contains tools to rewrite CSS retrieved from an online source so that it +can safely operate within a ZIM, linking only to ZIM entries everytime a URL is used. + +The rewriter needs to have an article url rewriter to rewrite URLs found in CSS, an +optional base href if the CSS to rewrite was found inline an HTML document which has a +base href set, and an optional flag indicating if in case of parsing error we want to +fallback to simple regex rewriting or we prefer to drop the offending rule. +""" + +import re +from collections.abc import Iterable +from functools import partial +from typing import Any + +from tinycss2 import ( + ast, + parse_declaration_list, # pyright: ignore[reportUnknownVariableType] + parse_stylesheet, # pyright: ignore[reportUnknownVariableType] + parse_stylesheet_bytes, # pyright: ignore[reportUnknownVariableType] + serialize, # pyright: ignore[reportUnknownVariableType] +) +from tinycss2.serializer import ( + serialize_url, # pyright: ignore[reportUnknownVariableType] +) + +from zimscraperlib import logger +from zimscraperlib.rewriting.rx_replacer import RxRewriter, TransformationRule +from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter + + +class FallbackRegexCssRewriter(RxRewriter): + """ + Fallback CSS rewriting based on regular expression. + + This is obviously way less powerful than real CSS parsing, but it allows to cope + with CSS we failed to parse without dropping any CSS rule (problem could be just a + parsing issue, not necessarily a bad CSS rule) + """ + + def __simple_transform( + self, + m_object: re.Match[str], + _opts: dict[str, Any] | None, + url_rewriter: ArticleUrlRewriter, + base_href: str | None, + ) -> str: + """Function to apply the regex rule""" + return "".join( + [ + "url(", + m_object["quote"], + url_rewriter(m_object["url"], base_href), + m_object["quote"], + ")", + ] + ) + + def __init__(self, url_rewriter: ArticleUrlRewriter, base_href: str | None): + """Create a RxRewriter adapted for CSS rules rewriting""" + + # we have only only rule, searching for url(...) functions and rewriting the + # URL found + rules = [ + TransformationRule( + [ + re.compile( + r"""url\((?P['"])?(?P.+?)(?P=quote)(? str: + """ + Rewrite a 'standalone' CSS document + + 'standalone' means "not inline an HTML document" + """ + try: + if isinstance(content, bytes): + rules, _ = ( # pyright: ignore[reportUnknownVariableType] + parse_stylesheet_bytes(content) + ) + + else: + rules = parse_stylesheet( # pyright: ignore[reportUnknownVariableType] + content + ) + self._process_list(rules) # pyright: ignore[reportUnknownArgumentType] + + return self._serialize_rules( + rules # pyright: ignore[reportUnknownArgumentType] + ) + except Exception: + # If tinycss fail to parse css, it will generate a "Error" token. + # Exception is raised at serialization time. + # We try/catch the whole process to be sure anyway. + logger.warning( + ( + "Css transformation fails. Fallback to regex rewriter.\n" + "Article path is %s" + ), + self.url_rewriter.article_url, + ) + return self.fallback_rewriter.rewrite(content, {}) + + def rewrite_inline(self, content: str) -> str: + """ + Rewrite an 'inline' CSS document + + 'inline' means "inline an HTML document" + """ + try: + rules = ( # pyright: ignore[reportUnknownVariableType] + parse_declaration_list(content) + ) + self._process_list(rules) # pyright: ignore[reportUnknownArgumentType] + return self._serialize_rules( + rules # pyright: ignore[reportUnknownArgumentType] + ) + except Exception: + # If tinycss fail to parse css, it will generate a "Error" token. + # Exception is raised at serialization time. + # We try/catch the whole process to be sure anyway. + logger.warning( + ( + "Css transformation fails. Fallback to regex rewriter.\n" + "Content is `%s`" + ), + content, + ) + return self.fallback_rewriter.rewrite(content, {}) + + def _process_list(self, nodes: Iterable[ast.Node] | None): + """Process a list of CSS nodes""" + if not nodes: + return + for node in nodes: + self._process_node(node) + + def _process_node(self, node: ast.Node): + """Process one single CSS node""" + if isinstance( + node, + ast.QualifiedRule + | ast.SquareBracketsBlock + | ast.ParenthesesBlock + | ast.CurlyBracketsBlock, + ): + self._process_list( + node.content, # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] + ) + elif isinstance(node, ast.FunctionBlock): + if node.lower_name == "url": # pyright: ignore[reportUnknownMemberType] + url_node: ast.Node = node.arguments[0] # pyright: ignore + new_url = self.url_rewriter( + url_node.value, # pyright: ignore + self.base_href, + ) + url_node.value = str(new_url) # pyright: ignore + url_node.representation = ( # pyright: ignore + f'"{serialize_url(str(new_url))}"' + ) + + else: + self._process_list( + node.arguments, # pyright: ignore + ) + elif isinstance(node, ast.AtRule): + self._process_list(node.prelude) # pyright: ignore + self._process_list(node.content) # pyright: ignore + elif isinstance(node, ast.Declaration): + self._process_list(node.value) # pyright: ignore + elif isinstance(node, ast.URLToken): + new_url = self.url_rewriter(node.value, self.base_href) # pyright: ignore + node.value = new_url + node.representation = f"url({serialize_url(new_url)})" + + def _serialize_rules(self, rules: list[ast.Node]) -> str: + """Serialize back all CSS rules to a string""" + return serialize( + [ + rule + for rule in rules + if not self.remove_errors or not isinstance(rule, ast.ParseError) + ] + ) diff --git a/src/zimscraperlib/rewriting/html.py b/src/zimscraperlib/rewriting/html.py new file mode 100644 index 00000000..3696aabb --- /dev/null +++ b/src/zimscraperlib/rewriting/html.py @@ -0,0 +1,712 @@ +""" HTML Rewriting + +This modules contains tools to rewrite HTML retrieved from an online source so that it +can safely operate within a ZIM. + +In addition to fixing links so that they point to ZIM item when it exists, it also fixes +a bunch of other tags which needs special handling. + +The rewriter needs to have an article url rewriter to rewrite URLs found in HTML, an +optional pre_head and post_head HTML code to insert (typically to load wombat.js in +pre_head and to load additional custom CSS in post_head), and an optional callable that +will be invoked everytime JS code file is encountered (useful to know which JS file is +classic and which is a module). +""" + +import io +import re +from collections.abc import Callable +from dataclasses import dataclass +from functools import cache +from html import escape +from html.parser import HTMLParser +from inspect import Signature, signature +from typing import Any, NamedTuple + +from bs4 import BeautifulSoup + +from zimscraperlib.rewriting.css import CssRewriter +from zimscraperlib.rewriting.js import JsRewriter +from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter, ZimPath + +AttrNameAndValue = tuple[str, str | None] +AttrsList = list[AttrNameAndValue] + +HTTP_EQUIV_REDIRECT_RE = re.compile( + r"^\s*(?P.*?)\s*;\s*url\s*=\s*(?P.*?)\s*$" +) + + +class RewritenHtml(NamedTuple): + """Result of rewrite operation""" + + title: str + content: str + + +def get_attr_value_from( + attrs: AttrsList, name: str, default: str | None = None +) -> str | None: + """Get one HTML attribute value if present, else return default value""" + for attr_name, value in attrs: + if attr_name == name: + return value + return default + + +def format_attr(name: str, value: str | None) -> str: + """Format a given attribute name and value, properly escaping the value""" + if value is None: + return name + html_escaped_value = escape(value, quote=True) + return f'{name}="{html_escaped_value}"' + + +def get_html_rewrite_context(tag: str, attrs: AttrsList) -> str: + """Get current HTML rewrite context + + By default, rewrite context is the HTML tag. But in some cases (e.g. script tags) we + need to be more precise since rewriting logic will vary based on another attribute + value (e.g. type attribute for script tags) + """ + if tag == "script": + script_type = get_attr_value_from(attrs, "type") + return { + "application/json": "json", + "json": "json", + "module": "js-module", + "application/javascript": "js-classic", + "text/javascript": "js-classic", + "": "js-classic", + }.get(script_type or "", "unknown") + elif tag == "link": + link_rel = get_attr_value_from(attrs, "rel") + if link_rel == "modulepreload": + return "js-module" + elif link_rel == "preload": + preload_type = get_attr_value_from(attrs, "as") + if preload_type == "script": + return "js-classic" + return tag + + +def extract_base_href(content: str) -> str | None: + """Extract base href value from HTML content + + This is done in a specific function before real parsing / rewriting of any HTML + because we need this information before rewriting any link since we might have stuff + before the tag in html head (e.g. for favicons) + """ + soup = BeautifulSoup(content, features="lxml") + if not soup.head: + return None + for base in soup.head.find_all("base"): + if base.has_attr("href"): + return base["href"] + return None + + +@cache +def _cached_signature(func: Callable[..., Any]) -> Signature: + """Returns the signature of a given callable + + Result is cached to save performance when reused multiple times + """ + return signature(func) + + +class HtmlRewriter(HTMLParser): + """ + HTML Rewriter to process HTML code and adapt it to work inside a ZIM. + + This class is extensible thanks to the module `rules` which can be used to decorate + any method which will handle some HTML rewriting. + + So far, following rules kinds are supported (see HtmlRewritingRules): + drop_attribute (to completely drop an HTML tag attribute), rewrite_attribute (to + modify the value of an HTML tag attribute), rewrite_tag (to modify a whole HTML tag, + typically modifying attributes names and values, rewrite_data (to rewrite the data/ + content of an HTML tag) + """ + + def __init__( + self, + url_rewriter: ArticleUrlRewriter, + pre_head_insert: str, + post_head_insert: str | None, + notify_js_module: Callable[[ZimPath], None], + ): + super().__init__(convert_charrefs=False) + self.url_rewriter = url_rewriter + self.title = None + self.output = None + # This works only for tag without children. + # But as we use it to get the title, we are ok + self.html_rewrite_context = None + self.pre_head_insert = pre_head_insert + self.post_head_insert = post_head_insert + self.notify_js_module = notify_js_module + + def rewrite(self, content: str) -> RewritenHtml: + """Rewrite HTML code passed""" + if self.output is not None: + raise Exception("ouput should not already be set") # pragma: no cover + self.output = io.StringIO() + + self.base_href = extract_base_href(content) + self.css_rewriter = CssRewriter(self.url_rewriter, self.base_href) + self.js_rewriter = JsRewriter( + url_rewriter=self.url_rewriter, + base_href=self.base_href, + notify_js_module=self.notify_js_module, + ) + + self.feed(content) + self.close() + + output = self.output.getvalue() + self.output = None + return RewritenHtml(self.title or "", output) + + def send(self, value: str): + """Overwrite send from HTMLParser""" + self.output.write(value) # pyright: ignore[reportOptionalMemberAccess] + + def handle_starttag(self, tag: str, attrs: AttrsList, *, auto_close: bool = False): + """Overwrite handle_starttag from HTMLParser""" + self.html_rewrite_context = get_html_rewrite_context(tag=tag, attrs=attrs) + + if ( + rewritten := rules.do_tag_rewrite( + tag=tag, attrs=attrs, auto_close=auto_close + ) + ) is not None: + self.send(rewritten) + return + + self.send(f"<{tag}") + if attrs: + self.send(" ") + self.send( + " ".join( + format_attr(*attr) + for attr in ( + rules.do_attribute_rewrite( + tag=tag, + attr_name=attr_name, + attr_value=attr_value, + attrs=attrs, + js_rewriter=self.js_rewriter, + css_rewriter=self.css_rewriter, + url_rewriter=self.url_rewriter, + base_href=self.base_href, + notify_js_module=self.notify_js_module, + ) + for attr_name, attr_value in attrs + if not rules.do_drop_attribute( + tag=tag, attr_name=attr_name, attr_value=attr_value, attrs=attrs + ) + ) + ) + ) + + if auto_close: + self.send(" />") + else: + self.send(">") + if tag == "head" and self.pre_head_insert: + self.send(self.pre_head_insert) + + def handle_endtag(self, tag: str): + """Overwrite handle_endtag from HTMLParser""" + self.html_rewrite_context = None + if tag == "head" and self.post_head_insert: + self.send(self.post_head_insert) + self.send(f"") + + def handle_startendtag(self, tag: str, attrs: AttrsList): + """Overwrite handle_startendtag from HTMLParser""" + self.handle_starttag(tag, attrs, auto_close=True) + self.html_rewrite_context = None + + def handle_data(self, data: str): + """Overwrite handle_data from HTMLParser""" + if self.html_rewrite_context == "title" and self.title is None: + self.title = data.strip() + if ( + data.strip() + and ( + rewritten := rules.do_data_rewrite( + html_rewrite_context=self.html_rewrite_context, + data=data, + css_rewriter=self.css_rewriter, + js_rewriter=self.js_rewriter, + url_rewriter=self.url_rewriter, + ) + ) + is not None + ): + self.send(rewritten) + return + self.send(data) + + def handle_entityref(self, name: str): + """Overwrite handle_entityref from HTMLParser""" + self.send(f"&{name};") + + def handle_charref(self, name: str): + """Overwrite handle_charref from HTMLParser""" + self.send(f"&#{name};") + + def handle_comment(self, data: str): + """Overwrite handle_comment from HTMLParser""" + self.send(f"") + + def handle_decl(self, decl: str): + """Overwrite handle_decl from HTMLParser""" + self.send(f"") + + def handle_pi(self, data: str): + """Overwrite handle_pi from HTMLParser""" + self.send(f"") + + def unknown_decl(self, data: str): + """Overwrite unknown_decl from HTMLParser""" + self.handle_decl(data) # pragma: no cover + + +DropAttributeCallable = Callable[..., bool] +RewriteAttributeCallable = Callable[..., AttrNameAndValue | None] +RewriteTagCallable = Callable[..., str | None] +RewriteDataCallable = Callable[..., str | None] + + +@dataclass(frozen=True) +class DropAttributeRule: + """A rule specifying when an HTML attribute should be dropped""" + + func: DropAttributeCallable + + +@dataclass(frozen=True) +class RewriteAttributeRule: + """A rule specifying how a given HTML attribute should be rewritten""" + + func: RewriteAttributeCallable + + +@dataclass(frozen=True) +class RewriteTagRule: + """A rule specifying how a given HTML tag should be rewritten""" + + func: RewriteTagCallable + + +@dataclass(frozen=True) +class RewriteDataRule: + """A rule specifying how a given HTML data should be rewritten""" + + func: RewriteDataCallable + + +def _check_decorated_func_signature( + expected_func: Callable[..., Any], decorated_func: Callable[..., Any] +): + """Checks if the decorated function signature is compatible + + It checks that decorated function parameters have known names and proper types + """ + expected_params = _cached_signature(expected_func).parameters + func_params = _cached_signature(decorated_func).parameters + for name, param in func_params.items(): + if name not in expected_params: + raise TypeError( + f"Parameter '{name}' is unsupported in function " + f"'{decorated_func.__name__}'" + ) + + if expected_params[name].annotation != param.annotation: + raise TypeError( + f"Parameter '{name}' in function '{decorated_func.__name__}' must be of" + f" type '{expected_params[name].annotation}'" + ) + + +class HTMLRewritingRules: + """A class holding the definitions of all rules to rewrite HTML documents""" + + def __init__(self) -> None: + self.drop_attribute_rules: set[DropAttributeRule] = set() + self.rewrite_attribute_rules: set[RewriteAttributeRule] = set() + self.rewrite_tag_rules: set[RewriteTagRule] = set() + self.rewrite_data_rules: set[RewriteDataRule] = set() + + def drop_attribute( + self, + ) -> Callable[[DropAttributeCallable], DropAttributeCallable]: + """Decorator to use when defining a rule regarding attribute dropping""" + + def decorator(func: DropAttributeCallable) -> DropAttributeCallable: + _check_decorated_func_signature(self.do_drop_attribute, func) + self.drop_attribute_rules.add(DropAttributeRule(func=func)) + return func + + return decorator + + def rewrite_attribute( + self, + ) -> Callable[[RewriteAttributeCallable], RewriteAttributeCallable]: + """Decorator to use when defining a rule regarding attribute rewriting""" + + def decorator(func: RewriteAttributeCallable) -> RewriteAttributeCallable: + _check_decorated_func_signature(self.do_attribute_rewrite, func) + self.rewrite_attribute_rules.add(RewriteAttributeRule(func=func)) + return func + + return decorator + + def rewrite_tag( + self, + ) -> Callable[[RewriteTagCallable], RewriteTagCallable]: + """Decorator to use when defining a rule regarding tag rewriting + + This has to be used when we need to rewrite the whole start tag. It can also + handle rewrites of startend tags (autoclosing tags). + """ + + def decorator(func: RewriteTagCallable) -> RewriteTagCallable: + _check_decorated_func_signature(self.do_tag_rewrite, func) + self.rewrite_tag_rules.add(RewriteTagRule(func=func)) + return func + + return decorator + + def rewrite_data( + self, + ) -> Callable[[RewriteDataCallable], RewriteDataCallable]: + """Decorator to use when defining a rule regarding data rewriting + + This has to be used when we need to rewrite the tag data. + """ + + def decorator(func: RewriteDataCallable) -> RewriteDataCallable: + _check_decorated_func_signature(self.do_data_rewrite, func) + self.rewrite_data_rules.add(RewriteDataRule(func=func)) + return func + + return decorator + + def do_drop_attribute( + self, tag: str, attr_name: str, attr_value: str | None, attrs: AttrsList + ) -> bool: + """Utility function to process all attribute dropping rules + + Returns true if at least one rule is matching + """ + return any( + rule.func( + **{ + arg_name: arg_value + for arg_name, arg_value in { # pyright: ignore[reportUnknownVariableType] + "tag": tag, + "attr_name": attr_name, + "attr_value": attr_value, + "attrs": attrs, + }.items() + if arg_name in _cached_signature(rule.func).parameters + } + ) + is True + for rule in self.drop_attribute_rules + ) + + def do_attribute_rewrite( + self, + tag: str, + attr_name: str, + attr_value: str | None, + attrs: AttrsList, + js_rewriter: JsRewriter, + css_rewriter: CssRewriter, + url_rewriter: ArticleUrlRewriter, + base_href: str | None, + notify_js_module: Callable[[ZimPath], None], + ) -> AttrNameAndValue: + """Utility function to process all attribute rewriting rules + + Returns the rewritten attribute name and value + """ + + if attr_value is None: + return attr_name, None + + for rule in self.rewrite_attribute_rules: + if ( + rewritten := rule.func( + **{ + arg_name: arg_value + for arg_name, arg_value in { # pyright: ignore[reportUnknownVariableType] + "tag": tag, + "attr_name": attr_name, + "attr_value": attr_value, + "attrs": attrs, + "js_rewriter": js_rewriter, + "css_rewriter": css_rewriter, + "url_rewriter": url_rewriter, + "base_href": base_href, + "notify_js_module": notify_js_module, + }.items() + if arg_name in _cached_signature(rule.func).parameters + } + ) + ) is not None: + attr_name, attr_value = rewritten + + return attr_name, attr_value + + def do_tag_rewrite( + self, + tag: str, + attrs: AttrsList, + *, + auto_close: bool, + ) -> str | None: + """Utility function to process all tag rewriting rules + + Returns the rewritten tag + """ + + for rule in self.rewrite_tag_rules: + if ( + rewritten := rule.func( + **{ + arg_name: arg_value + for arg_name, arg_value in { # pyright: ignore[reportUnknownVariableType] + "tag": tag, + "attrs": attrs, + "auto_close": auto_close, + }.items() + if arg_name in _cached_signature(rule.func).parameters + } + ) + ) is not None: + return rewritten + + def do_data_rewrite( + self, + html_rewrite_context: str | None, + data: str, + css_rewriter: CssRewriter, + js_rewriter: JsRewriter, + url_rewriter: ArticleUrlRewriter, + ) -> str | None: + """Utility function to process all data rewriting rules + + Returns the rewritten data + """ + + for rule in self.rewrite_data_rules: + if ( + rewritten := rule.func( + **{ + arg_name: arg_value + for arg_name, arg_value in { # pyright: ignore[reportUnknownVariableType] + "html_rewrite_context": html_rewrite_context, + "data": data, + "css_rewriter": css_rewriter, + "js_rewriter": js_rewriter, + "url_rewriter": url_rewriter, + }.items() + if arg_name in _cached_signature(rule.func).parameters + } + ) + ) is not None: + return rewritten + + +rules = HTMLRewritingRules() + + +@rules.drop_attribute() +def drop_script_integrity_attribute(tag: str, attr_name: str): + """Drop integrity attribute in + + +{% endautoescape %} + + diff --git a/src/zimscraperlib/rewriting/url_rewriting.py b/src/zimscraperlib/rewriting/url_rewriting.py new file mode 100644 index 00000000..fbf0147d --- /dev/null +++ b/src/zimscraperlib/rewriting/url_rewriting.py @@ -0,0 +1,424 @@ +""" URL rewriting tools + +This module is about url and entry path rewriting. + +The global scheme is the following: + +Entries are stored in the ZIM file using their decoded fully decoded path: +- The full path is the full url without the scheme, username, password, port, fragment + (ie : "/(? None: + HttpUrl.check_validity(value) + self._value = value + + def __eq__(self, __value: object) -> bool: + return isinstance(__value, HttpUrl) and __value.value == self.value + + def __hash__(self) -> int: + return self.value.__hash__() + + def __str__(self) -> str: + return f"HttpUrl({self.value})" + + def __repr__(self) -> str: + return f"{self.__str__} - {super().__repr__()}" # pragma: no cover + + @property + def value(self) -> str: + return self._value + + @classmethod + def check_validity(cls, value: str) -> None: + parts = urlsplit(value) + + if parts.scheme.lower() not in ["http", "https"]: + raise ValueError( + f"Incorrect HttpUrl scheme in value: {value} {parts.scheme}" + ) + + if not parts.hostname: + raise ValueError(f"Unsupported empty hostname in value: {value}") + + if parts.hostname.lower() not in value: + raise ValueError(f"Unsupported upper-case chars in hostname : {value}") + + +class ZimPath: + """A utility class representing a ZIM path, usefull to pass this data around + + Includes a basic validation, ensuring that path does start with scheme, hostname,... + """ + + def __init__(self, value: str) -> None: + ZimPath.check_validity(value) + self._value = value + + def __eq__(self, __value: object) -> bool: + return isinstance(__value, ZimPath) and __value.value == self.value + + def __hash__(self) -> int: + return self.value.__hash__() + + def __str__(self) -> str: + return f"ZimPath({self.value})" + + def __repr__(self) -> str: + return f"{self.__str__} - {super().__repr__()}" # pragma: no cover + + @property + def value(self) -> str: + return self._value + + @classmethod + def check_validity(cls, value: str) -> None: + parts = urlsplit(value) + + if parts.scheme: + raise ValueError(f"Unexpected scheme in value: {value} {parts.scheme}") + + if parts.hostname: + raise ValueError(f"Unexpected hostname in value: {value} {parts.hostname}") + + if parts.username: + raise ValueError(f"Unexpected username in value: {value} {parts.username}") + + if parts.password: + raise ValueError(f"Unexpected password in value: {value} {parts.password}") + + +class ArticleUrlRewriter: + """ + Rewrite urls in article. + + This is typically used to rewrite urls found in an HTML document, but can be used + beyong that usage. + """ + + additional_rules: ClassVar[list[AdditionalRule]] = COMPILED_FUZZY_RULES + + def __init__( + self, + *, + article_url: HttpUrl, + article_path: ZimPath | None = None, + existing_zim_paths: set[ZimPath] | None = None, + missing_zim_paths: set[ZimPath] | None = None, + ): + """ + Initialise the rewriter + + Args: + article_url: URL where the original document was located, used to resolve + relative URLS which will be passed + existing_zim_paths: list of ZIM paths which are known to exist, useful if one + wants to rewrite the URL to a local one only if item exists in the ZIM + missing_zim_paths: list of ZIM paths which are known to already be missing + from the existing_zim_paths ; usefull only in complement with this variable ; + new missing entries will be added as URLs are normalized in this function + + Results: + items_to_download: populated with the list of rewritten URLs, so that one + might use it to download items after rewriting the document + """ + self.article_path = article_path or ArticleUrlRewriter.normalize(article_url) + self.article_url = article_url + self.existing_zim_paths = existing_zim_paths + self.missing_zim_paths = missing_zim_paths + self.items_to_download: dict[ZimPath, HttpUrl] = {} + + def get_item_path(self, item_url: str, base_href: str | None) -> ZimPath: + """Utility to transform an item URL into a ZimPath""" + + item_absolute_url = urljoin( + urljoin(self.article_url.value, base_href), item_url + ) + return ArticleUrlRewriter.normalize(HttpUrl(item_absolute_url)) + + def __call__( + self, + item_url: str, + base_href: str | None, + *, + rewrite_all_url: bool = True, + ) -> str: + """Rewrite a url contained in a article. + + The url is "fully" rewrited to point to a normalized entry path + """ + + try: + item_url = item_url.strip() + + # Make case of standalone fragments more straightforward + if item_url.startswith("#"): + return item_url + + item_scheme = urlsplit(item_url).scheme + if item_scheme and item_scheme not in ("http", "https"): + return item_url + + item_absolute_url = urljoin( + urljoin(self.article_url.value, base_href), item_url + ) + + item_fragment = urlsplit(item_absolute_url).fragment + + item_path = ArticleUrlRewriter.normalize(HttpUrl(item_absolute_url)) + + if rewrite_all_url or ( + self.existing_zim_paths and item_path in self.existing_zim_paths + ): + if item_path not in self.items_to_download: + self.items_to_download[item_path] = HttpUrl(item_absolute_url) + return self.get_document_uri(item_path, item_fragment) + else: + if ( + self.missing_zim_paths is not None + and item_path not in self.missing_zim_paths + ): + logger.debug(f"WARNING {item_path} ({item_url}) not in archive.") + # maintain a collection of missing Zim Path to not fill the logs + # with duplicate messages + self.missing_zim_paths.add(item_path) + # The url doesn't point to a known entry + return item_absolute_url + + except Exception as exc: # pragma: no cover + item_scheme = ( + item_scheme # pyright: ignore[reportPossiblyUnboundVariable] + if "item_scheme" in locals() + else "" + ) + item_absolute_url = ( + item_absolute_url # pyright: ignore[reportPossiblyUnboundVariable] + if "item_absolute_url" in locals() + else "" + ) + item_fragment = ( + item_fragment # pyright: ignore[reportPossiblyUnboundVariable] + if "item_fragment" in locals() + else "" + ) + item_path = ( + item_path # pyright: ignore[reportPossiblyUnboundVariable] + if "item_path" in locals() + else "" + ) + logger.debug( + f"Invalid URL value found in {self.article_url.value}, kept as-is. " + f"(item_url: {item_url}, " + f"item_scheme: {item_scheme}, " + f"item_absolute_url: {item_absolute_url}, " + f"item_fragment: {item_fragment}, " + f"item_path: {item_path}, " + f"rewrite_all_url: {rewrite_all_url}", + exc_info=exc, + ) + return item_url + + def get_document_uri(self, item_path: ZimPath, item_fragment: str) -> str: + """Given an ZIM item path and its fragment, get the URI to use in document + + This function transforms the path of a ZIM item we want to adress from current + document (HTML / JS / ...) and returns the corresponding URI to use. + + It computes the relative path based on current document location and escape + everything which needs to be to transform the ZIM path into a valid RFC 3986 URI + + It also append a potential trailing item fragment at the end of the resulting + URI. + + """ + item_parts = urlsplit(item_path.value) + + # item_path is both path + querystring, both will be url-encoded in the document + # so that readers consider them as a whole and properly pass them to libzim + item_url = item_parts.path + if item_parts.query: + item_url += "?" + item_parts.query + relative_path = str( + PurePosixPath(item_url).relative_to( + ( + PurePosixPath(self.article_path.value) + if self.article_path.value.endswith("/") + else PurePosixPath(self.article_path.value).parent + ), + walk_up=True, + ) + ) + # relative_to removes a potential last '/' in the path, we add it back + if item_path.value.endswith("/"): + relative_path += "/" + + return ( + f"{quote(relative_path, safe='/')}" + f"{'#' + item_fragment if item_fragment else ''}" + ) + + @classmethod + def apply_additional_rules(cls, uri: HttpUrl | str) -> str: + """Apply additional rules on a URL or relative path + + First matching additional rule matching the input value is applied and its + result is returned. + + If no additional rule is matching, the input is returned as-is. + """ + value = uri.value if isinstance(uri, HttpUrl) else uri + for rule in cls.additional_rules: + if match := rule.match.match(value): + return match.expand(rule.replace) + return value + + @classmethod + def normalize(cls, url: HttpUrl) -> ZimPath: + """Transform a HTTP URL into a ZIM path to use as a entry's key. + + According to RFC 3986, a URL allows only a very limited set of characters, so we + assume by default that the url is encoded to match this specification. + + The transformation rewrites the hostname, the path and the querystring. + + The transformation drops the URL scheme, username, password, port and fragment: + - we suppose there is no conflict of URL scheme or port: there is no two + ressources with same hostname, path and querystring but different URL scheme or + port leading to different content + - we consider username/password port are purely authentication mechanism which + have no impact on the content to server + - we know that the fragment is never passed to the server, it stays in the + User-Agent, so if we encounter a fragment while normalizing a URL found in a + document, it won't make its way to the ZIM anyway and will stay client-side + + The transformation consists mainly in decoding the three components so that ZIM + path is not encoded at all, as required by the ZIM specification. + + Decoding is done differently for the hostname (decoded with puny encoding) and + the path and querystring (both decoded with url decoding). + + The final transformation is the application of fuzzy rules (sourced from wabac) + to transform some URLs into replay URLs and drop some useless stuff. + + Returned value is a ZIM path, without any puny/url encoding applied, ready to be + passed to python-libzim for UTF-8 encoding. + """ + + if not isinstance(url, HttpUrl): + raise ValueError("Bad argument type passed, HttpUrl expected") + + url_parts = urlsplit(url.value) + + if not url_parts.hostname: + # cannot happen because of the HttpUrl checks, but important to please the + # type checker + raise Exception("Hostname is missing") # pragma: no cover + + # decode the hostname if it is punny-encoded + hostname = ( + idna.decode(url_parts.hostname) + if url_parts.hostname.startswith("xn--") + else url_parts.hostname + ) + + path = url_parts.path + + if path: + # unquote the path so that it is stored unencoded in the ZIM as required by + # ZIM specification + path = unquote(path) + else: + # if path is empty, we need a "/" to remove ambiguities, e.g. + # https://example.com and https://example.com/ must all lead to the same ZIM + # entry to match RFC 3986 section 6.2.3: + # https://www.rfc-editor.org/rfc/rfc3986#section-6.2.3 + path = "/" + + query = url_parts.query + + # if query is missing, we do not add it at all, not even a trailing ? without + # anything after it + if url_parts.query: + # `+`` in query parameter must be decoded as space first to remove + # ambiguities between a space (encoded as `+` in url query parameter) and a + # real plus sign (encoded as %2B but soon decoded in ZIM path) + query = query.replace("+", " ") + # unquote the query so that it is stored unencoded in the ZIM as required by + # ZIM specification + query = "?" + unquote(query) + else: + query = "" + + fuzzified_url = ArticleUrlRewriter.apply_additional_rules( + f"{hostname}{ArticleUrlRewriter._remove_subsequent_slashes(path)}{ArticleUrlRewriter._remove_subsequent_slashes(query)}" + ) + + return ZimPath(fuzzified_url) + + @classmethod + def _remove_subsequent_slashes(cls, value: str) -> str: + """Remove all successive occurence of a slash `/` in a given string + + E.g `val//ue` or `val///ue` or `val////ue` (and so on) are transformed into + `value` + """ + return re.sub(r"//+", "/", value) diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index c20357c8..02cae889 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -16,10 +16,9 @@ import io from collections import namedtuple -from typing import Dict MimetypeAndCounter = namedtuple("MimetypeAndCounter", ["mimetype", "value"]) -CounterMap = Dict[ +CounterMap = dict[ type(MimetypeAndCounter.mimetype), type(MimetypeAndCounter.value) # pyright: ignore ] diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 0dab5029..a4558b8d 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -264,7 +264,7 @@ def convert_and_check_metadata( Also checks that final type is appropriate for libzim (str or bytes) """ - if name == "Date" and isinstance(value, (datetime.date, datetime.datetime)): + if name == "Date" and isinstance(value, datetime.date | datetime.datetime): value = value.strftime("%Y-%m-%d") if ( name == "Tags" diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index a7625b07..e1f9e9b2 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -129,7 +129,7 @@ def get_contentprovider(self) -> libzim.writer.ContentProvider: # content was set manually content = getattr(self, "content", None) if content is not None: - if not isinstance(content, (str, bytes)): + if not isinstance(content, str | bytes): raise AttributeError(f"Unexpected type for content: {type(content)}") return StringProvider(content=content, ref=self) @@ -155,7 +155,7 @@ def _get_auto_index(self): # content was set manually content = getattr(self, "content", None) if content is not None: - if not isinstance(content, (str, bytes)): + if not isinstance(content, str | bytes): raise RuntimeError( f"Unexpected type for content: {type(content)}" ) # pragma: no cover diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 3db12c7b..411e42f6 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -60,7 +60,7 @@ def validate_title(name: str, value: str): def validate_date(name: str, value: datetime.datetime | datetime.date | str): """ensures Date metadata can be casted to an ISO 8601 string""" if name == "Date": - if not isinstance(value, (datetime.datetime, datetime.date, str)): + if not isinstance(value, datetime.datetime | datetime.date | str): raise ValueError(f"Invalid type for {name}: {type(value)}") elif isinstance(value, str): match = re.match(r"(?P\d{4})-(?P\d{2})-(?P\d{2})", value) diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 2c384ddb..a4748cbb 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -13,7 +13,7 @@ import io import pathlib -from typing import Generator +from collections.abc import Generator import libzim.writer # pyright: ignore import requests diff --git a/tests/rewriting/__init__.py b/tests/rewriting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/rewriting/conftest.py b/tests/rewriting/conftest.py new file mode 100644 index 00000000..390dd471 --- /dev/null +++ b/tests/rewriting/conftest.py @@ -0,0 +1,100 @@ +from collections.abc import Callable, Generator + +import pytest + +from zimscraperlib.rewriting.css import CssRewriter +from zimscraperlib.rewriting.js import JsRewriter +from zimscraperlib.rewriting.url_rewriting import ( + ArticleUrlRewriter, + HttpUrl, + ZimPath, +) + + +@pytest.fixture(scope="module") +def no_js_notify(): + """Fixture to not care about notification of detection of a JS file""" + + def no_js_notify_handler(_: str): + pass + + yield no_js_notify_handler + + +class SimpleUrlRewriter(ArticleUrlRewriter): + """Basic URL rewriter mocking most calls""" + + def __init__(self, article_url: HttpUrl, suffix: str = ""): + self.article_url = article_url + self.suffix = suffix + + def __call__( + self, + item_url: str, + base_href: str | None, # noqa: ARG002 + *, + rewrite_all_url: bool = True, # noqa: ARG002 + ) -> str: + return item_url + self.suffix + + def get_item_path( + self, item_url: str, base_href: str | None # noqa: ARG002 + ) -> ZimPath: + return ZimPath("") + + def get_document_uri( + self, item_path: ZimPath, item_fragment: str # noqa: ARG002 + ) -> str: + return "" + + +@pytest.fixture(scope="module") +def simple_url_rewriter_gen() -> ( + Generator[Callable[[str], ArticleUrlRewriter], None, None] +): + """Fixture to create a basic url rewriter returning URLs as-is""" + + def get_simple_url_rewriter(url: str, suffix: str = "") -> ArticleUrlRewriter: + return SimpleUrlRewriter(HttpUrl(url), suffix=suffix) + + yield get_simple_url_rewriter + + +@pytest.fixture(scope="module") +def js_rewriter_gen() -> Generator[ + Callable[[ArticleUrlRewriter, str | None, Callable[[ZimPath], None]], JsRewriter], + None, + None, +]: + """Fixture to create a basic url rewriter returning URLs as-is""" + + def get_js_rewriter( + url_rewriter: ArticleUrlRewriter, + base_href: str | None, + notify_js_module: Callable[[ZimPath], None], + ) -> JsRewriter: + return JsRewriter( + url_rewriter=url_rewriter, + base_href=base_href, + notify_js_module=notify_js_module, + ) + + yield get_js_rewriter + + +@pytest.fixture(scope="module") +def css_rewriter_gen() -> ( + Generator[Callable[[ArticleUrlRewriter, str | None], CssRewriter], None, None] +): + """Fixture to create a basic url rewriter returning URLs as-is""" + + def get_css_rewriter( + url_rewriter: ArticleUrlRewriter, + base_href: str | None, + ) -> CssRewriter: + return CssRewriter( + url_rewriter=url_rewriter, + base_href=base_href, + ) + + yield get_css_rewriter diff --git a/tests/rewriting/test_css_rewriting.py b/tests/rewriting/test_css_rewriting.py new file mode 100644 index 00000000..a43ce849 --- /dev/null +++ b/tests/rewriting/test_css_rewriting.py @@ -0,0 +1,218 @@ +from textwrap import dedent + +import pytest + +from zimscraperlib.rewriting.css import CssRewriter +from zimscraperlib.rewriting.url_rewriting import ArticleUrlRewriter, HttpUrl + +from .utils import ContentForTests + + +@pytest.fixture( + params=[ + ContentForTests(input_=b"p { color: red; }"), + ContentForTests(input_=b"p {\n color: red;\n}"), + ContentForTests(input_=b"p { background: blue; }"), + ContentForTests(input_=b"p { background: rgb(15, 0, 52); }"), + ContentForTests( + input_=b"/* See bug issue at http://exemple.com/issue/link */ " + b"p { color: blue; }" + ), + ContentForTests( + input_=b"p { width= } div { background: url(http://exemple.com/img.png)}", + expected=b"p { width= } div { background: url(../exemple.com/img.png)}", + ), + ContentForTests( + input_=b"p { width= } div { background: url('http://exemple.com/img.png')}", + expected=b'p { width= } div { background: url("../exemple.com/img.png")}', + ), + ContentForTests( + input_=b'p { width= } div { background: url("http://exemple.com/img.png")}', + expected=b'p { width= } div { background: url("../exemple.com/img.png")}', + ), + ] +) +def no_rewrite_content(request: pytest.FixtureRequest): + yield request.param + + +def test_no_rewrite(no_rewrite_content: ContentForTests): + assert ( + CssRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{no_rewrite_content.article_url}") + ), + base_href=None, + ).rewrite(no_rewrite_content.input_bytes) + == no_rewrite_content.expected_bytes.decode() + ) + + +def test_no_rewrite_str(): + test_css = "p {\n color: red;\n}" + assert ( + CssRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org")), + base_href=None, + ).rewrite(test_css) + == test_css + ) + + +@pytest.fixture( + params=[ + ContentForTests(input_='"border:'), + ContentForTests(input_="border: solid 1px #c0c0c0; width= 100%"), + # Despite being invalid, tinycss parse it as "width" property without value. + ContentForTests(input_="width:", expected="width:;"), + ContentForTests( + input_="border-bottom-width: 1px;border-bottom-color: #c0c0c0;w" + ), + ContentForTests( + input_='background: url("http://exemple.com/foo.png"); width=', + expected='background: url("../exemple.com/foo.png"); width=', + ), + ] +) +def invalid_content_inline_with_fallback(request: pytest.FixtureRequest): + yield request.param + + +def test_invalid_css_inline_with_fallback( + invalid_content_inline_with_fallback: ContentForTests, +): + assert ( + CssRewriter( + ArticleUrlRewriter( + article_url=HttpUrl( + f"http://{invalid_content_inline_with_fallback.article_url}" + ) + ), + base_href=None, + ).rewrite_inline(invalid_content_inline_with_fallback.input_str) + == invalid_content_inline_with_fallback.expected_str + ) + + +@pytest.fixture( + params=[ + ContentForTests(input_='"border:', expected=""), + ContentForTests( + input_="border: solid 1px #c0c0c0; width= 100%", + expected="border: solid 1px #c0c0c0; ", + ), + # Despite being invalid, tinycss parse it as "width" property without value. + ContentForTests(input_="width:", expected="width:;"), + ContentForTests( + input_="border-bottom-width: 1px;border-bottom-color: #c0c0c0;w", + expected="border-bottom-width: 1px;border-bottom-color: #c0c0c0;", + ), + ContentForTests( + input_='background: url("http://exemple.com/foo.png"); width=', + expected='background: url("../exemple.com/foo.png"); ', + ), + ] +) +def invalid_content_inline_no_fallback(request: pytest.FixtureRequest): + yield request.param + + +def test_invalid_css_inline_no_fallback( + invalid_content_inline_no_fallback: ContentForTests, +): + assert ( + CssRewriter( + ArticleUrlRewriter( + article_url=HttpUrl( + f"http://{invalid_content_inline_no_fallback.article_url}" + ) + ), + base_href=None, + remove_errors=True, + ).rewrite_inline(invalid_content_inline_no_fallback.input_str) + == invalid_content_inline_no_fallback.expected_str + ) + + +@pytest.fixture( + params=[ + # Tinycss parse `"border:}` as a string with an unexpected eof in string. + # At serialization, tiny try to recover and close the opened rule + ContentForTests(input_=b'p {"border:}', expected=b'p {"border:}}'), + ContentForTests(input_=b'"p {border:}'), + ContentForTests(input_=b"p { border: solid 1px #c0c0c0; width= 100% }"), + ContentForTests(input_=b"p { width: }"), + ContentForTests( + input_=b"p { border-bottom-width: 1px;border-bottom-color: #c0c0c0;w }" + ), + ContentForTests( + input_=b'p { background: url("http://exemple.com/foo.png"); width= }', + expected=b'p { background: url("../exemple.com/foo.png"); width= }', + ), + ] +) +def invalid_content(request: pytest.FixtureRequest): + yield request.param + + +def test_invalid_cssl(invalid_content: ContentForTests): + assert ( + CssRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{invalid_content.article_url}") + ), + base_href=None, + ).rewrite(invalid_content.input_bytes) + == invalid_content.expected_bytes.decode() + ) + + +def test_rewrite(): + content = b""" +/* A comment with a link : http://foo.com */ +@import url(//fonts.googleapis.com/icon?family=Material+Icons); + +p, input { + color: rbg(1, 2, 3); + background: url('http://kiwix.org/super/img'); + background-image:url('http://exemple.com/no_space_before_url'); +} + +@font-face { + src: url(https://f.gst.com/s/qa/v31/6xKtdSZaE8KbpRA_hJFQNcOM.woff2) format('woff2'); +} + +@media only screen and (max-width: 40em) { + p, input { + background-image:url(data:image/png;base64,FooContent); + } +}""" + + expected = """ + /* A comment with a link : http://foo.com */ + @import url(../fonts.googleapis.com/icon%3Ffamily%3DMaterial%20Icons); + + p, input { + color: rbg(1, 2, 3); + background: url("super/img"); + background-image:url("../exemple.com/no_space_before_url"); + } + + @font-face { + src: url(../f.gst.com/s/qa/v31/6xKtdSZaE8KbpRA_hJFQNcOM.woff2) format("woff2"); + } + + @media only screen and (max-width: 40em) { + p, input { + background-image:url(data:image/png;base64,FooContent); + } + }""" + expected = dedent(expected) + + assert ( + CssRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/article")), + base_href=None, + ).rewrite(content) + == expected + ) diff --git a/tests/rewriting/test_html_rewriting.py b/tests/rewriting/test_html_rewriting.py new file mode 100644 index 00000000..bd59b497 --- /dev/null +++ b/tests/rewriting/test_html_rewriting.py @@ -0,0 +1,1555 @@ +from collections.abc import Callable +from textwrap import dedent + +import pytest + +from zimscraperlib.rewriting.css import CssRewriter +from zimscraperlib.rewriting.html import ( + AttrNameAndValue, + AttrsList, + HtmlRewriter, + HTMLRewritingRules, + extract_base_href, + format_attr, + get_attr_value_from, + rewrite_meta_http_equiv_redirect, +) +from zimscraperlib.rewriting.js import JsRewriter +from zimscraperlib.rewriting.url_rewriting import ( + ArticleUrlRewriter, + HttpUrl, + ZimPath, +) + +from .utils import ContentForTests + + +@pytest.fixture( + params=[ + ContentForTests(input_="A simple string without url"), + ContentForTests( + input_="" + "

This is a sentence with a http://exemple.com/path link

" + "" + ), + ContentForTests( + input_='A link not to rewrite' + ), + ContentForTests( + input_='

A url (relative) in a ' + "inline style

" + ), + ContentForTests(input_="

"), + ContentForTests( + input_="' + ), + ContentForTests(input_=""), + ContentForTests( + input_="""This is a sample attribute with a quote """ + "in its value and which is not a URL" + ), + ContentForTests(input_=""), + ContentForTests(input_="<script>"), + ContentForTests( + input_="

This is a smiley (🙂) and it html hex value (🙂)

" + ), + ContentForTests( + input_='' + ), + ContentForTests( + input_='" + ), + ContentForTests( + input_='' + ), + ContentForTests(input_="A simple string with doctype"), + ContentForTests(input_="A simple string with comment"), + ContentForTests(input_="A simple string with pi"), + ] +) +def no_rewrite_content(request: pytest.FixtureRequest): + yield request.param + + +def test_no_rewrite( + no_rewrite_content: ContentForTests, no_js_notify: Callable[[ZimPath], None] +): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{no_rewrite_content.article_url}"), + ), + "", + "", + no_js_notify, + ) + .rewrite(no_rewrite_content.input_str) + .content + == no_rewrite_content.expected_str + ) + + +@pytest.fixture( + params=[ + ContentForTests( + "

A link in a inline style" + "

", + '

' + "A link in a inline style

", + ), + ContentForTests( + "

" + "A link in a inline style

", + '

' + "A link in a inline style

", + ), + ContentForTests( + "
    ", + '
      ', + ), + ] +) +def escaped_content(request: pytest.FixtureRequest): + yield request.param + + +def test_escaped_content( + escaped_content: ContentForTests, no_js_notify: Callable[[ZimPath], None] +): + transformed = ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{escaped_content.article_url}") + ), + "", + "", + no_js_notify, + ) + .rewrite(escaped_content.input_str) + .content + ) + assert transformed == escaped_content.expected_str + + +@pytest.fixture( + params=[ + ContentForTests( + '', + ( + "" + ), + ), + ContentForTests( + '', + ( + """" + ), + ), + ContentForTests( + '', + ( + """" + ), + ), + ContentForTests( + '', + '', + ), + ContentForTests( + '', + '', + ), + ContentForTests( + '', + '', + ), + ] +) +def js_rewrites(request: pytest.FixtureRequest): + yield request.param + + +def test_js_rewrites( + js_rewrites: ContentForTests, no_js_notify: Callable[[ZimPath], None] +): + transformed = ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{js_rewrites.article_url}") + ), + "", + "", + no_js_notify, + ) + .rewrite(js_rewrites.input_str) + .content + ) + assert transformed == js_rewrites.expected_str + + +def long_path_replace_test_content(input_: str, rewriten_url: str, article_url: str): + expected = input_.replace("http://exemple.com/a/long/path", rewriten_url) + return ContentForTests(input_, expected, article_url) + + +lprtc = long_path_replace_test_content + + +@pytest.fixture( + params=[ + # Normalized path is "exemple.com/a/long/path" + lprtc( + 'A link to rewrite', + "a/long/path", + "exemple.com", + ), + lprtc( + 'A link to rewrite', + "../exemple.com/a/long/path", + "kiwix.org", + ), + lprtc( + 'A link to rewrite', + "../exemple.com/a/long/path", + "kiwix.org/", + ), + lprtc( + 'A link to rewrite', + "a/long/path", + "exemple.com/", + ), + lprtc( + 'A link to rewrite', + "a/long/path", + "exemple.com/a", + ), + lprtc( + 'A link to rewrite', + "long/path", + "exemple.com/a/", + ), + lprtc( + 'A link to rewrite', + "long/path", + "exemple.com/a/long", + ), + lprtc( + 'A link to rewrite', + "path", + "exemple.com/a/long/", + ), + lprtc( + 'A link to rewrite', + "path", + "exemple.com/a/long/path", + ), + lprtc( + 'A link to rewrite', + ".", + "exemple.com/a/long/path/yes", + ), + lprtc( + 'A link to rewrite', + "../../long/path", + "exemple.com/a/very/long/path", + ), + lprtc( + 'A link to rewrite', + "../../exemple.com/a/long/path", + "kiwix.org/another/path", + ), + ] +) +def rewrite_url(request: pytest.FixtureRequest): + yield request.param + + +def test_rewrite(rewrite_url: ContentForTests, no_js_notify: Callable[[ZimPath], None]): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{rewrite_url.article_url}"), + existing_zim_paths={ZimPath("exemple.com/a/long/path")}, + ), + "", + "", + no_js_notify, + ) + .rewrite(rewrite_url.input_str) + .content + == rewrite_url.expected_str + ) + + +def test_extract_title(no_js_notify: Callable[[ZimPath], None]): + content = """ + + Page title + + + Wrong page title + + """ + + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl("http://example.com"), + existing_zim_paths={ZimPath("exemple.com/a/long/path")}, + ), + "", + "", + no_js_notify, + ) + .rewrite(content) + .title + == "Page title" + ) + + +def test_rewrite_attributes(no_js_notify: Callable[[ZimPath], None]): + rewriter = HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/"), + existing_zim_paths={ZimPath("kiwix.org/foo")}, + ), + "", + "", + no_js_notify, + ) + + assert ( + rewriter.rewrite("A link").content + == 'A link' + ) + + assert ( + rewriter.rewrite("").content + == '' + ) + + assert ( + rewriter.rewrite( + "" + ).content + == '' + ) + + +def test_rewrite_css(no_js_notify: Callable[[ZimPath], None]): + output = ( + HtmlRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/")), + "", + "", + no_js_notify, + ) + .rewrite( + "", + ) + .content + ) + assert ( + output == "' + ) + + +def test_head_insert(no_js_notify: Callable[[ZimPath], None]): + content = """ + + A test content + + + """ + + content = dedent(content) + + url_rewriter = ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/")) + assert ( + HtmlRewriter(url_rewriter, "", "", no_js_notify).rewrite(content).content + == content + ) + + assert HtmlRewriter(url_rewriter, "PRE_HEAD_INSERT", "", no_js_notify).rewrite( + content + ).content == content.replace("", "PRE_HEAD_INSERT") + assert HtmlRewriter(url_rewriter, "", "POST_HEAD_INSERT", no_js_notify).rewrite( + content + ).content == content.replace("", "POST_HEAD_INSERT") + assert HtmlRewriter( + url_rewriter, "PRE_HEAD_INSERT", "POST_HEAD_INSERT", no_js_notify + ).rewrite(content).content == content.replace( + "", "PRE_HEAD_INSERT" + ).replace( + "", "POST_HEAD_INSERT" + ) + + +@pytest.mark.parametrize( + "js_src,expected_js_module_path", + [ + ("my-module-script.js", "kiwix.org/my_folder/my-module-script.js"), + ("./my-module-script.js", "kiwix.org/my_folder/my-module-script.js"), + ("../my-module-script.js", "kiwix.org/my-module-script.js"), + ("../../../my-module-script.js", "kiwix.org/my-module-script.js"), + ("/my-module-script.js", "kiwix.org/my-module-script.js"), + ("//myserver.com/my-module-script.js", "myserver.com/my-module-script.js"), + ( + "https://myserver.com/my-module-script.js", + "myserver.com/my-module-script.js", + ), + ], +) +def test_js_module_detected_script(js_src: str, expected_js_module_path: str): + + js_modules = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + HtmlRewriter( + url_rewriter=ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ), + pre_head_insert="", + post_head_insert="", + notify_js_module=custom_notify, + ).rewrite(f'') + + assert len(js_modules) == 1 + assert js_modules[0].value == expected_js_module_path + + +@pytest.mark.parametrize( + "js_src,expected_js_module_path", + [ + ("my-module-script.js", "kiwix.org/my_folder/my-module-script.js"), + ("./my-module-script.js", "kiwix.org/my_folder/my-module-script.js"), + ("../my-module-script.js", "kiwix.org/my-module-script.js"), + ("../../../my-module-script.js", "kiwix.org/my-module-script.js"), + ("/my-module-script.js", "kiwix.org/my-module-script.js"), + ("//myserver.com/my-module-script.js", "myserver.com/my-module-script.js"), + ( + "https://myserver.com/my-module-script.js", + "myserver.com/my-module-script.js", + ), + ], +) +def test_js_module_detected_module_preload(js_src: str, expected_js_module_path: str): + + js_modules = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + HtmlRewriter( + url_rewriter=ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ), + pre_head_insert="", + post_head_insert="", + notify_js_module=custom_notify, + ).rewrite(f'') + + assert len(js_modules) == 1 + assert js_modules[0].value == expected_js_module_path + + +@pytest.mark.parametrize( + "script_src", + [ + (''), + (''), + (''), + ], +) +def test_no_js_module_detected(script_src: str): + + js_modules = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + HtmlRewriter( + url_rewriter=ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ), + pre_head_insert="", + post_head_insert="", + notify_js_module=custom_notify, + ).rewrite(script_src) + + assert len(js_modules) == 0 + + +def test_js_module_base_href_src(): + + js_modules = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + HtmlRewriter( + url_rewriter=ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ), + pre_head_insert="", + post_head_insert="", + notify_js_module=custom_notify, + ).rewrite( + """ + + """ + ) + + assert len(js_modules) == 1 + assert js_modules[0].value == "kiwix.org/my_other_folder/my-module-script.js" + + +def test_js_module_base_href_inline(): + + js_modules = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + HtmlRewriter( + url_rewriter=ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ), + pre_head_insert="", + post_head_insert="", + notify_js_module=custom_notify, + ).rewrite( + """ + + """ + ) + + assert len(js_modules) == 1 + assert js_modules[0].value == "kiwix.org/my_other_folder/my-module-script.js" + + +@pytest.mark.parametrize( + "html_content, expected_base_href", + [ + pytest.param("", None, id="empty_content"), + pytest.param("", None, id="empty_html"), + pytest.param( + "Foo", None, id="no_base" + ), + pytest.param( + '', "../..", id="standard_case" + ), + pytest.param( + '', "../..", id="malformed_head" + ), # malformed HTML is OK + pytest.param( + '', "../..", id="very_malformed_head" + ), # even very malformed HTML is OK + pytest.param( + '', "../..", id="base_at_root" + ), # even very malformed HTML is OK + pytest.param( + '', None, id="base_in_body" + ), # but base in body is ignored + pytest.param( + '', + "../..", + id="base_with_target_before", + ), + pytest.param( + '', + "../..", + id="base_with_target_after", + ), + pytest.param( + '', + "../..", + id="base_with_two_href", + ), + pytest.param( + '', + "../..", + id="two_bases_with_href", + ), + pytest.param( + '', + "../..", + id="href_in_second_base", + ), + pytest.param( + '' + "", + "../..", + id="href_in_second_base_second_href_ignored", + ), + ], +) +def test_extract_base_href(html_content: str, expected_base_href: str): + assert extract_base_href(html_content) == expected_base_href + + +@pytest.fixture( + params=[ + ContentForTests( + input_='' + '', + expected='', + ), + ContentForTests( + '' + '', + '', + "kiwix.org/a/index.html", + ), + ContentForTests( + '' + '', + '' + '', + ), + ContentForTests( + '' + '', + '' + '', + ), + ContentForTests( + '' + '', + '' + '', + ), + ContentForTests( + '' + '', + '' + '', + ), + ContentForTests( + '' + "" + '', + '' + '', + ), + ContentForTests( + '' + '', + '', + "kiwix.org/a/index.html", + ), + ContentForTests( + '' + '', + '' + "", + "kiwix.org/a/index.html", + ), + ContentForTests( + ' ' + '', + ' ' + "", + "kiwix.org/a/index.html", + ), + ] +) +def rewrite_base_href_content(request): + yield request.param + + +def test_rewrite_base_href( + rewrite_base_href_content: ContentForTests, no_js_notify: Callable[[ZimPath], None] +): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{rewrite_base_href_content.article_url}"), + existing_zim_paths={ + ZimPath("kiwix.org/foo.html"), + ZimPath("kiwix.org/foo.js"), + ZimPath("kiwix.org/foo.css"), + ZimPath("kiwix.org/foo.png"), + ZimPath("kiwix.org/favicon.png"), + }, + ), + "", + "", + no_js_notify, + ) + .rewrite(rewrite_base_href_content.input_str) + .content + == rewrite_base_href_content.expected_str + ) + + +@pytest.mark.parametrize( + "input_content,expected_output", + [ + pytest.param( + """""", + """""", + id="double_quoted_attr", + ), + pytest.param( + "", + """""", + id="single_quoted_attr", + ), + pytest.param( + """""", + """""", + id="uppercase_named_reference_in_attr", + ), + pytest.param( + """""", + """""", + id="numeric_reference_in_attr", + ), + pytest.param( + """""", + """""", + id="numeric_reference_in_attr", + ), + pytest.param( + """""", + """""", + id="badly_escaped_src", + ), + ], +) +def test_simple_rewrite( + input_content: str, expected_output: str, no_js_notify: Callable[[ZimPath], None] +): + assert ( + HtmlRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://example.com")), + "", + "", + no_js_notify, + ) + .rewrite(input_content) + .content + == expected_output + ) + + +@pytest.fixture( + params=[ + ContentForTests( + """""", + ), + ContentForTests( + """""", + ), + ContentForTests( + """""", + ), + ContentForTests( + """""", + ), + ContentForTests( + """""", + ( + """""" + ), # NOTA: quotes and ampersand are escaped since we are inside HTML attr + ), + ] +) +def rewrite_onxxx_content(request: pytest.FixtureRequest): + yield request.param + + +def test_rewrite_onxxx_event( + rewrite_onxxx_content: ContentForTests, no_js_notify: Callable[[ZimPath], None] +): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl(f"http://{rewrite_onxxx_content.article_url}"), + existing_zim_paths={ + ZimPath("kiwix.org/foo.html"), + ZimPath("kiwix.org/foo.js"), + ZimPath("kiwix.org/foo.css"), + ZimPath("kiwix.org/foo.png"), + ZimPath("kiwix.org/favicon.png"), + }, + ), + "", + "", + no_js_notify, + ) + .rewrite(rewrite_onxxx_content.input_str) + .content + == rewrite_onxxx_content.expected_str + ) + + +@pytest.fixture( + params=[ + ContentForTests( + 'whatever', + ), + ContentForTests( + '' + "whatever", + 'whatever', + ), + ContentForTests( + "" + '' + "whatever", + ), + ContentForTests( + "" + '' + "whatever", + "" + '' + "whatever", + ), + ContentForTests( + 'whatever', + ), # do not rewrite other tags mentionning a charset + ContentForTests( + "" + '' + "whatever", + ), # do not rewrite other http-equiv mentionning a charset + ] +) +def rewrite_meta_charset_content(request: pytest.FixtureRequest): + yield request.param + + +def test_rewrite_meta_charset( + rewrite_meta_charset_content: ContentForTests, + no_js_notify: Callable[[ZimPath], None], +): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl( + f"http://{rewrite_meta_charset_content.article_url}" + ) + ), + "", + "", + no_js_notify, + ) + .rewrite(rewrite_meta_charset_content.input_str) + .content + == rewrite_meta_charset_content.expected_str + ) + + +@pytest.fixture( + params=[ + ContentForTests( + '' + "whatever", + '' + "whatever", + ), + ] +) +def rewrite_meta_http_equiv_redirect_full_content(request: pytest.FixtureRequest): + yield request.param + + +def test_rewrite_meta_http_equiv_redirect_full( + rewrite_meta_http_equiv_redirect_full_content: ContentForTests, + no_js_notify: Callable[[ZimPath], None], +): + assert ( + HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl( + f"http://{rewrite_meta_http_equiv_redirect_full_content.article_url}" + ), + existing_zim_paths={ZimPath("kiwix.org/somepage")}, + ), + "", + "", + no_js_notify, + ) + .rewrite(rewrite_meta_http_equiv_redirect_full_content.input_str) + .content + == rewrite_meta_http_equiv_redirect_full_content.expected_str + ) + + +rules = HTMLRewritingRules() + + +@rules.drop_attribute() +def drop_all_named_attribute(attr_name: str): + return attr_name == "all_named" + + +@rules.drop_attribute() +def drop_all_tag_name_attribute(tag: str): + return tag == "all_tag" + + +@rules.drop_attribute() +def drop_tag_name_attribute(tag: str, attr_name: str): + return tag == "drop_tag" and attr_name == "drop_name" + + +@rules.drop_attribute() +def drop_attr_name_and_value_attribute(attr_name: str, attr_value: str | None): + return ( + attr_name == "drop_value" + and attr_value is not None + and attr_value.startswith("drop") + ) + + +@rules.drop_attribute() +def drop_if_other_attribute(attr_name: str, attrs: AttrsList): + return attr_name == "drop_if_other" and any( + other_name == "other" for other_name, _ in attrs + ) + + +@pytest.mark.parametrize( + "tag, attr_name, attr_value, attrs, should_drop", + [ + pytest.param("all_tag", "foo", "bar", [], True, id="drop_by_tag_name"), + pytest.param("other_tag", "foo", "bar", [], False, id="dont_drop_by_tag_name"), + pytest.param("foo", "all_named", "bar", [], True, id="drop_by_attr_name"), + pytest.param( + "foo", "other_name", "bar", [], False, id="dont_drop_by_attr_name" + ), + pytest.param( + "drop_tag", "drop_name", "bar", [], True, id="drop_by_tag_and_attr_name" + ), + pytest.param( + "drop_tag", "foo", "bar", [], False, id="dont_drop_by_tag_and_attr_name" + ), + pytest.param("foo", "drop_value", "drop_me", [], True, id="drop_by_attr_value"), + pytest.param( + "foo", "drop_value", "dont_drop", [], False, id="dont_drop_by_attr_value" + ), + pytest.param( + "foo", "drop_value", "dont_drop", [], False, id="dont_drop_by_attr_value" + ), + pytest.param( + "foo", + "drop_if_other", + "bar", + [("foo", None), ("other", "foo"), ("bar", "foo")], + True, + id="drop_if_other", + ), + pytest.param( + "foo", + "drop_if_other", + "bar", + [("foo", None), ("bar", "foo")], + False, + id="dont_drop_if_not_other", + ), + ], +) +def test_html_drop_rules( + tag: str, + attr_name: str, + attr_value: str | None, + attrs: AttrsList, + *, + should_drop: bool, +): + assert ( + rules.do_drop_attribute( + tag=tag, attr_name=attr_name, attr_value=attr_value, attrs=attrs + ) + is should_drop + ) + + +def test_bad_html_drop_rules_argument_name(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* is unsupported in function"): + + @bad_rules.drop_attribute() + def bad_signature(foo: str) -> bool: + return foo == "bar" + + +def test_bad_html_drop_rules_argument_type(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* in function .* must be of type"): + + @bad_rules.drop_attribute() + def bad_signature( # pyright: ignore[reportUnusedFunction] + attr_name: int, + ) -> bool: + return attr_name == 123 + + +@rules.rewrite_attribute() +def rewrite_tag_value(attr_name: str) -> AttrNameAndValue | None: + if attr_name != "aname": + return + return (attr_name, "foo") + + +@rules.rewrite_attribute() +def rewrite_tag_name(attr_name: str, attr_value: str | None) -> AttrNameAndValue | None: + if attr_name != "bad_name": + return + return ("good_name", attr_value) + + +@rules.rewrite_attribute() +def rewrite_call_notify( + attr_name: str, + notify_js_module: Callable[[ZimPath], None], +) -> AttrNameAndValue | None: + if attr_name != "call_notify": + return + notify_js_module(ZimPath("foo")) + return + + +@rules.rewrite_attribute() +def rewrite_value_with_base_href( + attr_name: str, + base_href: str | None, +) -> AttrNameAndValue | None: + if attr_name != "get_base_href": + return + return (attr_name, base_href) + + +@rules.rewrite_attribute() +def rewrite_attr2_value_with_attr1_value( + attr_name: str, + attrs: AttrsList, +) -> AttrNameAndValue | None: + if attr_name != "attr2": + return + return (attr_name, get_attr_value_from(attrs, "attr1")) + + +@pytest.mark.parametrize( + "tag, attr_name, attr_value, attrs, base_href, expected_result, should_notify", + [ + pytest.param( + "foo", + "aname", + "bar", + [], + "", + ("aname", "foo"), + False, + id="rewrite_tag_value", + ), + pytest.param( + "foo", + "bad_name", + "bar", + [], + "", + ("good_name", "bar"), + False, + id="rewrite_tag_name", + ), + pytest.param( + "foo", + "call_notify", + "bar", + [], + "", + ("call_notify", "bar"), + True, + id="call_notify", + ), + pytest.param( + "foo", + "get_base_href", + "bar", + [], + "base_href_value", + ("get_base_href", "base_href_value"), + False, + id="rewrite_value_with_base_href", + ), + pytest.param( + "foo", + "attr2", + "bar", + [("attr1", "value1")], + "base_href_value", + ("attr2", "value1"), + False, + id="rewrite_attr2_value_with_attr1_value", + ), + ], +) +def test_html_attribute_rewrite_rules( + tag: str, + attr_name: str, + attr_value: str | None, + attrs: AttrsList, + base_href: str, + expected_result: AttrNameAndValue, + *, + should_notify: bool, + simple_url_rewriter_gen: Callable[[str], ArticleUrlRewriter], + js_rewriter_gen: Callable[ + [ArticleUrlRewriter, str | None, Callable[[ZimPath], None]], JsRewriter + ], + css_rewriter_gen: Callable[[ArticleUrlRewriter, str | None], CssRewriter], +): + notified_paths: list[ZimPath] = [] + + def notify(path: ZimPath): + notified_paths.append(path) + + url_rewriter = simple_url_rewriter_gen("http://www.example.com") + js_rewriter = js_rewriter_gen(url_rewriter, base_href, notify) + css_rewriter = css_rewriter_gen(url_rewriter, base_href) + + assert ( + rules.do_attribute_rewrite( + tag=tag, + attr_name=attr_name, + attr_value=attr_value, + attrs=attrs, + js_rewriter=js_rewriter, + css_rewriter=css_rewriter, + url_rewriter=url_rewriter, + base_href=base_href, + notify_js_module=notify, + ) + == expected_result + ) + assert (len(notified_paths) > 0) == should_notify + + +def test_bad_html_attribute_rewrite_rules_argument_name(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* is unsupported in function"): + + @bad_rules.rewrite_attribute() + def bad_signature( # pyright: ignore[reportUnusedFunction] + foo: str, + ) -> AttrNameAndValue | None: + return (foo, "bar") + + +def test_bad_html_attribute_rewrite_rules_argument_type(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* in function .* must be of type"): + + @bad_rules.rewrite_attribute() + def bad_signature( # pyright: ignore[reportUnusedFunction] + attr_name: int, + ) -> AttrNameAndValue | None: + return (f"{attr_name}", "bar") + + +@rules.rewrite_tag() +def rewrite1_tag( + tag: str, +) -> str | None: + if tag != "rewrite1": + return + return "" + + +@rules.rewrite_tag() +def rewrite2_tag( + tag: str, + attrs: AttrsList, + *, + auto_close: bool, +) -> str | None: + if tag != "rewrite2": + return + + return ( + f"' if auto_close else '>'}" + ) + + +@pytest.mark.parametrize( + "tag, attrs, auto_close, expected_result", + [ + pytest.param( + "foo", + [], + False, + None, + id="do_not_rewrite_foo_tag", + ), + pytest.param( + "rewrite1", + [("attr2", "value2")], + False, + "", + id="rewrite1_tag", + ), + pytest.param( + "rewrite2", + [("attr2", "value2")], + False, + '', + id="rewrite2_tag_no_close", + ), + pytest.param( + "rewrite2", + [("attr2", "value2")], + True, + '', + id="rewrite2_tag_auto_close", + ), + ], +) +def test_html_tag_rewrite_rules( + tag: str, + attrs: AttrsList, + *, + auto_close: bool, + expected_result: str | None, +): + assert ( + rules.do_tag_rewrite( + tag=tag, + attrs=attrs, + auto_close=auto_close, + ) + == expected_result + ) + + +def test_bad_html_tag_rewrite_rules_argument_name(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* is unsupported in function"): + + @bad_rules.rewrite_tag() + def bad_signature(foo: str) -> str: # pyright: ignore[reportUnusedFunction] + return foo + + +def test_bad_html_tag_rewrite_rules_argument_type(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* in function .* must be of type"): + + @bad_rules.rewrite_tag() + def bad_signature(attrs: int) -> str: # pyright: ignore[reportUnusedFunction] + return f"{attrs}" + + +@rules.rewrite_data() +def rewrite_data_html_rewrite_context( + html_rewrite_context: str | None, +) -> str | None: + if html_rewrite_context != "rewrite": + return + return "rewritten data" + + +@pytest.mark.parametrize( + "html_rewrite_context, base_href, data, expected_result", + [ + pytest.param( + "foo", + "bar", + "something", + None, + id="do_not_rewrite_foo_context", + ), + pytest.param( + None, + "bar", + "something", + None, + id="do_not_rewrite_none_context", + ), + pytest.param( + "rewrite", + "bar", + "something", + "rewritten data", + id="rewrite_data_html_rewrite_context", + ), + ], +) +def test_html_data_rewrite_rules( + html_rewrite_context: str | None, + base_href: str, + data: str, + *, + expected_result: str | None, + simple_url_rewriter_gen: Callable[[str], ArticleUrlRewriter], + js_rewriter_gen: Callable[ + [ArticleUrlRewriter, str | None, Callable[[ZimPath], None]], JsRewriter + ], + css_rewriter_gen: Callable[[ArticleUrlRewriter, str | None], CssRewriter], +): + notified_paths: list[ZimPath] = [] + + def notify(path: ZimPath): + notified_paths.append(path) + + url_rewriter = simple_url_rewriter_gen("http://www.example.com") + js_rewriter = js_rewriter_gen(url_rewriter, base_href, notify) + css_rewriter = css_rewriter_gen(url_rewriter, base_href) + + assert ( + rules.do_data_rewrite( + html_rewrite_context=html_rewrite_context, + data=data, + css_rewriter=css_rewriter, + js_rewriter=js_rewriter, + url_rewriter=url_rewriter, + ) + == expected_result + ) + + +def test_bad_html_data_rewrite_rules_argument_name(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* is unsupported in function"): + + @bad_rules.rewrite_data() + def bad_signature( # pyright: ignore[reportUnusedFunction] + foo: str, + ) -> str | None: + return foo + + +def test_bad_html_data_rewrite_rules_argument_type(): + bad_rules = HTMLRewritingRules() + + with pytest.raises(TypeError, match="Parameter .* in function .* must be of type"): + + @bad_rules.rewrite_data() + def bad_signature( # pyright: ignore[reportUnusedFunction] + data: int, + ) -> str | None: + return f"{data}" + + +@pytest.mark.parametrize( + "tag, attr_name, attr_value, attrs, expected_result", + [ + pytest.param( + "meta", + "content", + "1;url=http://www.example.com/somewhere", + [("http-equiv", "refresh")], + ("content", "1;url=http://www.example.com/somewhererewritten"), + id="nomimal_case", + ), + pytest.param( + "meta", + "content", + " 1 ; url = http://www.example.com/somewhere ", + [("http-equiv", "refresh")], + ("content", "1;url=http://www.example.com/somewhererewritten"), + id="nomimal_case_with_spaces", + ), + pytest.param( + "foo", + "content", + "1;url=http://www.example.com/somewhere", + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_foo_tag", + ), + pytest.param( + "meta", + "foo", + "1;url=http://www.example.com/somewhere", + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_foo_attribute", + ), + pytest.param( + "meta", + "content", + "1;url=http://www.example.com/somewhere", + [("http-equiv", "foo")], + None, + id="do_not_rewrite_http_equiv_not_refresh", + ), + pytest.param( + "meta", + "content", + "1;url=http://www.example.com/somewhere", + [], + None, + id="do_not_rewrite_no_http_equiv", + ), + pytest.param( + "meta", + "content", + None, + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_missing_attribute", + ), + pytest.param( + "meta", + "content", + "", + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_empty_attribute", + ), + pytest.param( + "meta", + "content", + "1", + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_attribute_without_url", + ), + pytest.param( + "meta", + "content", + "1;foo=http://www.example.com/somewhere", + [("http-equiv", "refresh")], + None, + id="do_not_rewrite_bad_attribute", + ), + ], +) +def test_rewrite_meta_http_equiv_redirect_rule( + tag: str, + attr_name: str, + attr_value: str | None, + attrs: AttrsList, + expected_result: AttrNameAndValue | None, + simple_url_rewriter_gen: Callable[[str, str], ArticleUrlRewriter], +): + url_rewriter = simple_url_rewriter_gen("http://www.example.com", "rewritten") + + assert ( + rewrite_meta_http_equiv_redirect( + tag=tag, + attr_name=attr_name, + attr_value=attr_value, + attrs=attrs, + url_rewriter=url_rewriter, + base_href=None, + ) + == expected_result + ) diff --git a/tests/rewriting/test_js_rewriting.py b/tests/rewriting/test_js_rewriting.py new file mode 100644 index 00000000..1ee7ceea --- /dev/null +++ b/tests/rewriting/test_js_rewriting.py @@ -0,0 +1,402 @@ +from collections.abc import Callable + +import pytest + +from zimscraperlib.rewriting.js import JsRewriter +from zimscraperlib.rewriting.url_rewriting import ( + ArticleUrlRewriter, + HttpUrl, + ZimPath, +) + +from .utils import ContentForTests + + +@pytest.fixture +def simple_js_rewriter( + simple_url_rewriter_gen: Callable[[str], ArticleUrlRewriter], + no_js_notify: Callable[[ZimPath], None], +) -> JsRewriter: + return JsRewriter( + url_rewriter=simple_url_rewriter_gen("http://www.example.com"), + base_href=None, + notify_js_module=no_js_notify, + ) + + +@pytest.fixture( + params=[ + "a = this;", + "return this.location", + 'func(Function("return this"));', + "'a||this||that", + "(a,b,Q.contains(i[t], this))", + "a = this.location.href; exports.Foo = Foo; /* export className */", + ] +) +def rewrite_this_js_content(request: pytest.FixtureRequest): + content = request.param + yield ContentForTests( + input_=content, + expected=content.replace( + "this", "_____WB$wombat$check$this$function_____(this)" + ), + ) + + +def test_this_js_rewrite( + simple_js_rewriter: JsRewriter, rewrite_this_js_content: ContentForTests +): + assert ( + simple_js_rewriter.rewrite(rewrite_this_js_content.input_str) + == rewrite_this_js_content.expected_str + ) + + +@pytest.fixture( + params=[ + "aaa.this.window=red", + "aaa. this.window=red", + "aaa$this.window=red", + "a = this.color;", + "return this.color", + 'func(Function("return this.color"));', + "'a||this.color||that", + "(a,b,Q.contains(i[t], this.color))", + "a = this.color.href; exports.Foo = Foo; /* export className */", + ] +) +def no_rewrite_this_js_content(request: pytest.FixtureRequest): + content = request.param + yield ContentForTests(input_=content) + + +def test_this_no_js_rewrite( + simple_js_rewriter: JsRewriter, no_rewrite_this_js_content: ContentForTests +): + assert ( + simple_js_rewriter.rewrite(no_rewrite_this_js_content.input_str) + == no_rewrite_this_js_content.expected_str + ) + + +# This test probably has to be fixed but spec is blurry +# See https://github.com/openzim/warc2zim/issues/410 +def test_this_js_rewrite_newline(simple_js_rewriter: JsRewriter): + assert ( + simple_js_rewriter.rewrite("aaa\n this.window=red") + == "aaa\n ;_____WB$wombat$check$this$function_____(this).window=red" + ) + + +def test_js_rewrite_bytes_inline(simple_js_rewriter: JsRewriter): + assert ( + simple_js_rewriter.rewrite(b"a=123;\nb=456;", opts={"inline": True}) + == "a=123; b=456;" + ) + + +def test_js_rewrite_post_message(simple_js_rewriter: JsRewriter): + assert ( + simple_js_rewriter.rewrite(b"a.postMessage(") == "a.__WB_pmw(self).postMessage(" + ) + + +class WrappedTestContent(ContentForTests): + + def __init__( + self, + input_: str | bytes, + expected: str | bytes | None = None, + article_url: str = "https://kiwix.org", + ) -> None: + super().__init__(input_=input_, expected=expected, article_url=article_url) + self.expected = self.wrap_script(self.expected_str) + + @staticmethod + def wrap_script(text: str) -> str: + """ + A small wrapper to help generate the expected content. + + JsRewriter must add this local definition around all js code (when we access on + of the local varibles) + """ + return ( + "var _____WB$wombat$assign$function_____ = function(name) {return (self." + "_wb_wombat && self._wb_wombat.local_init && self._wb_wombat.local_init" + "(name)) || self[name]; };\n" + "if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source =" + " obj; return this; } }\n" + "{\n" + 'let window = _____WB$wombat$assign$function_____("window");\n' + 'let globalThis = _____WB$wombat$assign$function_____("globalThis");\n' + 'let self = _____WB$wombat$assign$function_____("self");\n' + 'let document = _____WB$wombat$assign$function_____("document");\n' + 'let location = _____WB$wombat$assign$function_____("location");\n' + 'let top = _____WB$wombat$assign$function_____("top");\n' + 'let parent = _____WB$wombat$assign$function_____("parent");\n' + 'let frames = _____WB$wombat$assign$function_____("frames");\n' + 'let opener = _____WB$wombat$assign$function_____("opener");\n' + "let arguments;\n" + "\n" + f"{text}" + "\n" + "}" + ) + + +@pytest.fixture( + params=[ + WrappedTestContent( + input_="location = http://example.com/", + expected="location = ((self.__WB_check_loc && " + "self.__WB_check_loc(location, argument" + "s)) || {}).href = http://example.com/", + ), + WrappedTestContent( + input_=" location = http://example.com/2", + expected=" location = ((self.__WB_check_loc && " + "self.__WB_check_loc(location, arguments)) || {}).href = " + "http://example.com/2", + ), + WrappedTestContent(input_="func(location = 0)", expected="func(location = 0)"), + WrappedTestContent( + input_=" location = http://example.com/2", + expected=" location = ((self.__WB_check_loc && " + "self.__WB_check_loc(location, arguments)) || {}).href = " + "http://example.com/2", + ), + WrappedTestContent(input_="window.eval(a)", expected="window.eval(a)"), + WrappedTestContent( + input_="x = window.eval; x(a);", expected="x = window.eval; x(a);" + ), + WrappedTestContent( + input_="this. location = 'http://example.com/'", + expected="this. location = 'http://example.com/'", + ), + WrappedTestContent( + input_="if (self.foo) { console.log('blah') }", + expected="if (self.foo) { console.log('blah') }", + ), + WrappedTestContent(input_="window.x = 5", expected="window.x = 5"), + ] +) +def rewrite_wrapped_content(request: pytest.FixtureRequest): + yield request.param + + +def test_wrapped_rewrite( + simple_js_rewriter: JsRewriter, rewrite_wrapped_content: WrappedTestContent +): + assert ( + simple_js_rewriter.rewrite(rewrite_wrapped_content.input_str) + == rewrite_wrapped_content.expected_str + ) + + +class ImportTestContent(ContentForTests): + + def __init__( + self, + input_: str | bytes, + expected: str | bytes | None = None, + article_url: str = "https://kiwix.org", + ) -> None: + super().__init__(input_=input_, expected=expected, article_url=article_url) + self.article_url = "https://exemple.com/some/path/" + self.expected = self.wrap_import(self.expected_str) + + @staticmethod + # We want to import js stored in zim file as `_zim_static/__wb_module_decl.js` from + # `https://exemple.com/some/path/` so path is + # `../../../_zim_static/__wb_module_decl.js` + def wrap_import(text: str) -> str: + """ + A small wrapper to help us generate the expected content for modules. + + JsRewriter must add this import line at beginning of module codes (when code + contains `import` or `export`) + """ + return ( + "import { window, globalThis, self, document, location, top, parent, " + 'frames, opener } from "../../../_zim_static/__wb_module_decl.js";\n' + f"{text}" + ) + + +@pytest.fixture( + params=[ + # import rewrite + ImportTestContent( + input_="""import "foo"; + +a = this.location""", + expected="""import "foo"; + +a = _____WB$wombat$check$this$function_____(this).location""", + ), + # import/export module rewrite + ImportTestContent( + input_="""a = this.location + +export { a }; +""", + expected="""a = _____WB$wombat$check$this$function_____(this).location + +export { a }; +""", + ), + # rewrite ESM module import + ImportTestContent( + input_='import "https://example.com/file.js"', + expected='import "../../../example.com/file.js"', + ), + ImportTestContent( + input_=''' +import {A, B} + from + "https://example.com/file.js"''', + expected=''' +import {A, B} + from + "../../../example.com/file.js"''', + ), + ImportTestContent( + input_=""" +import * from "https://example.com/file.js" +import A from "http://example.com/path/file2.js"; + +import {C, D} from "./abc.js"; +import {X, Y} from "../parent.js"; +import {E, F, G} from "/path.js"; +import { Z } from "../../../path.js"; + +B = await import(somefile); +""", + expected=""" +import * from "../../../example.com/file.js" +import A from "../../../example.com/path/file2.js"; + +import {C, D} from "./abc.js"; +import {X, Y} from "../parent.js"; +import {E, F, G} from "../../path.js"; +import { Z } from "../../path.js"; + +B = await ____wb_rewrite_import__(import.meta.url, somefile); +""", + ), + ImportTestContent( + input_='import"import.js";import{A, B, C} from"test.js";(function() => ' + '{ frames[0].href = "/abc"; })', + expected='import"import.js";import{A, B, C} from"test.js";(function() => ' + '{ frames[0].href = "/abc"; })', + ), + ImportTestContent( + input_="""a = location + +export{ a, $ as b}; +""", + expected="""a = location + +export{ a, $ as b}; +""", + ), + ] +) +def rewrite_import_content(request: pytest.FixtureRequest): + yield request.param + + +def test_import_rewrite( + no_js_notify: Callable[[ZimPath], None], rewrite_import_content: ImportTestContent +): + url_rewriter = ArticleUrlRewriter( + article_url=HttpUrl(rewrite_import_content.article_url) + ) + assert ( + JsRewriter( + url_rewriter=url_rewriter, base_href=None, notify_js_module=no_js_notify + ).rewrite(rewrite_import_content.input_str, opts={"isModule": True}) + == rewrite_import_content.expected_str + ) + + +@pytest.fixture( + params=[ + "return this.abc", + "return this object", + "a = 'some, this object'", + "{foo: bar, this: other}", + "this.$location = http://example.com/", + "this. $location = http://example.com/", + "this. _location = http://example.com/", + "this. alocation = http://example.com/", + "this.location = http://example.com/", + ",eval(a)", + "this.$eval(a)", + "x = $eval; x(a);", + "obj = { eval : 1 }", + "x = obj.eval", + "x = obj.eval(a)", + "x = obj._eval(a)", + "x = obj.$eval(a)", + "if (a.self.foo) { console.log('blah') }", + "a.window.x = 5", + " postMessage({'a': 'b'})", + "simport(5);", + "a.import(5);", + "$import(5);", + "async import(val) { ... }", + """function blah() { + const text = "text: import a from B.js"; +} +""", + """function blah() { + const text = ` +import a from "https://example.com/B.js" +`; +} + +""", + "let a = 7; var b = 5; const foo = 4;\n\n", + ] +) +def no_rewrite_js_content(request: pytest.FixtureRequest): + yield request.param + + +def test_no_rewrite(simple_js_rewriter: JsRewriter, no_rewrite_js_content: str): + assert simple_js_rewriter.rewrite(no_rewrite_js_content) == no_rewrite_js_content + + +@pytest.mark.parametrize( + "js_src,expected_js_module_path", + [ + ("./my-module-script.js", "kiwix.org/my_folder/my-module-script.js"), + ("../my-module-script.js", "kiwix.org/my-module-script.js"), + ("../../../my-module-script.js", "kiwix.org/my-module-script.js"), + ("/my-module-script.js", "kiwix.org/my-module-script.js"), + ("//myserver.com/my-module-script.js", "myserver.com/my-module-script.js"), + ( + "https://myserver.com/my-module-script.js", + "myserver.com/my-module-script.js", + ), + ], +) +def test_js_rewrite_nested_module_detected(js_src: str, expected_js_module_path: str): + + js_modules: list[ZimPath] = [] + + def custom_notify(zim_path: ZimPath): + js_modules.append(zim_path) + + url_rewriter = ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/my_folder/my_article.html") + ) + + JsRewriter( + url_rewriter=url_rewriter, base_href=None, notify_js_module=custom_notify + ).rewrite(f'import * from "{js_src}"', opts={"isModule": True}) + + assert len(js_modules) == 1 + assert js_modules[0].value == expected_js_module_path diff --git a/tests/rewriting/test_rx_replacer.py b/tests/rewriting/test_rx_replacer.py new file mode 100644 index 00000000..a2e24efe --- /dev/null +++ b/tests/rewriting/test_rx_replacer.py @@ -0,0 +1,107 @@ +import re +from collections.abc import Callable + +import pytest + +from zimscraperlib.rewriting.rx_replacer import ( + RxRewriter, + TransformationAction, + add_prefix, + add_suffix, + replace, + replace_all, + replace_prefix_from, +) + + +@pytest.fixture() +def compiled_rule() -> re.Pattern[str]: + return re.compile("") + + +@pytest.mark.parametrize( + "operation, operand1, expected_result", + [ + (add_suffix, "456", "pre456post"), + (add_prefix, "456", "pre456post"), + (replace_all, "456", "pre456post"), + ], +) +def test_actions_one( + compiled_rule: re.Pattern[str], + operation: Callable[[str], TransformationAction], + operand1: str, + expected_result: str, +): + def wrapped(operation: Callable[[str], TransformationAction], operand1: str): + def wraper(match: re.Match[str]): + return operation(operand1)(match, {}) + + return wraper + + assert ( + compiled_rule.sub(wrapped(operation, operand1), "prepost") + == expected_result + ) + + +@pytest.mark.parametrize( + "operation, operand1, operand2, expected_result", + [ + (replace_prefix_from, "456", "pl", "prepost"), + ], +) +def test_actions_two( + compiled_rule: re.Pattern[str], + operation: Callable[[str, str], TransformationAction], + operand1: str, + operand2: str, + expected_result: str, +): + def wrapped( + operation: Callable[[str, str], TransformationAction], + operand1: str, + operand2: str, + ): + def wraper(match: re.Match[str]): + return operation(operand1, operand2)(match, {}) + + return wraper + + assert ( + compiled_rule.sub(wrapped(operation, operand1, operand2), "prepost") + == expected_result + ) + + +@pytest.mark.parametrize( + "text, expected", + [ + ("prepost", "prepost"), + (b"prepost", "prepost"), + ("foo", "f456"), + ("bar", "bar"), + ("blu", "blu"), + ], +) +def test_rx_rewriter(text: str, expected: str): + rewriter = RxRewriter( + rules=[ + (re.compile("foo"), replace("oo", "456")), + (re.compile("bar"), replace("oo", "456")), + (re.compile(""), replace("pla", "123")), + ] + ) + assert rewriter.rewrite(text) == expected + + +def test_rx_rewriter_no_rules(): + rewriter = RxRewriter() + rewriter._compile_rules( + [ + (re.compile(""), replace("pla", "123")), + ] + ) + assert rewriter.rewrite("prepost") == "prepost" diff --git a/tests/rewriting/test_url_rewriting.py b/tests/rewriting/test_url_rewriting.py new file mode 100644 index 00000000..c14e9f84 --- /dev/null +++ b/tests/rewriting/test_url_rewriting.py @@ -0,0 +1,805 @@ +import pytest + +from zimscraperlib.rewriting.url_rewriting import ( + ArticleUrlRewriter, + HttpUrl, + ZimPath, +) + + +class TestNormalize: + + @pytest.mark.parametrize( + "url,zim_path", + [ + ("https://exemple.com", "exemple.com/"), + ("https://exemple.com/", "exemple.com/"), + ("http://example.com/resource", "example.com/resource"), + ("http://example.com/resource/", "example.com/resource/"), + ( + "http://example.com/resource/folder/sub.txt", + "example.com/resource/folder/sub.txt", + ), + ( + "http://example.com/resource/folder/sub", + "example.com/resource/folder/sub", + ), + ( + "http://example.com/resource/folder/sub?foo=bar", + "example.com/resource/folder/sub?foo=bar", + ), + ( + "http://example.com/resource/folder/sub?foo=bar#anchor1", + "example.com/resource/folder/sub?foo=bar", + ), + ("http://example.com/resource/#anchor1", "example.com/resource/"), + ("http://example.com/resource/?foo=bar", "example.com/resource/?foo=bar"), + ("http://example.com#anchor1", "example.com/"), + ("http://example.com?foo=bar#anchor1", "example.com/?foo=bar"), + ("http://example.com/?foo=bar", "example.com/?foo=bar"), + ("http://example.com/?foo=ba+r", "example.com/?foo=ba r"), + ( + "http://example.com/?foo=ba r", + "example.com/?foo=ba r", + ), # situation where the ` ` has not been properly escaped in document + ("http://example.com/?foo=ba%2Br", "example.com/?foo=ba+r"), + ("http://example.com/?foo=ba+%2B+r", "example.com/?foo=ba + r"), + ("http://example.com/#anchor1", "example.com/"), + ( + "http://example.com/some/path/http://example.com//some/path", + "example.com/some/path/http:/example.com/some/path", + ), + ( + "http://example.com/some/pa?th/http://example.com//some/path", + "example.com/some/pa?th/http:/example.com/some/path", + ), + ( + "http://example.com/so?me/pa?th/http://example.com//some/path", + "example.com/so?me/pa?th/http:/example.com/some/path", + ), + ("http://example.com/resource?", "example.com/resource"), + ("http://example.com/resource#", "example.com/resource"), + ("http://user@example.com/resource", "example.com/resource"), + ("http://user:password@example.com/resource", "example.com/resource"), + ("http://example.com:8080/resource", "example.com/resource"), + ( + "http://foobargooglevideo.com/videoplayback?id=1576&key=value", + "youtube.fuzzy.replayweb.page/videoplayback?id=1576", + ), # Fuzzy rule is applied in addition to path transformations + ("https://xn--exmple-cva.com", "exémple.com/"), + ("https://xn--exmple-cva.com/", "exémple.com/"), + ("https://xn--exmple-cva.com/resource", "exémple.com/resource"), + ("https://exémple.com/", "exémple.com/"), + ("https://exémple.com/resource", "exémple.com/resource"), + # host_ip is an invalid hostname according to spec + ("https://host_ip/", "host_ip/"), + ("https://host_ip/resource", "host_ip/resource"), + ("https://192.168.1.1/", "192.168.1.1/"), + ("https://192.168.1.1/resource", "192.168.1.1/resource"), + ("http://example.com/res%24urce", "example.com/res$urce"), + ( + "http://example.com/resource?foo=b%24r", + "example.com/resource?foo=b$r", + ), + ("http://example.com/resource@300x", "example.com/resource@300x"), + ("http://example.com:8080/resource", "example.com/resource"), + ("http://user@example.com:8080/resource", "example.com/resource"), + ("http://user:password@example.com:8080/resource", "example.com/resource"), + # the two URI below are an illustration of a potential collision (two + # differents URI leading to the same ZIM path) + ( + "http://tmp.kiwix.org/ci/test-website/images/urlencoding1_ico%CC%82ne-" + "de%CC%81buter-Solidarite%CC%81-Nume%CC%81rique_1%40300x.png", + "tmp.kiwix.org/ci/test-website/images/urlencoding1_icône-débuter-" + "Solidarité-Numérique_1@300x.png", + ), + ( + "https://tmp.kiwix.org/ci/test-website/images/urlencoding1_ico%CC%82ne-" + "de%CC%81buter-Solidarite%CC%81-Nume%CC%81rique_1@300x.png", + "tmp.kiwix.org/ci/test-website/images/urlencoding1_icône-débuter-" + "Solidarité-Numérique_1@300x.png", + ), + ], + ) + def test_normalize(self, url, zim_path): + assert ( + ArticleUrlRewriter.normalize(HttpUrl(url)).value == ZimPath(zim_path).value + ) + + def test_normalize_bad_arg( + self, + ): + with pytest.raises( + ValueError, match="Bad argument type passed, HttpUrl expected" + ): + ArticleUrlRewriter.normalize( + "https://www.acme.com" # pyright: ignore[reportArgumentType] + ) + + +class TestArticleUrlRewriter: + @pytest.mark.parametrize( + "original_content_url, expected_missing_zim_paths", + [ + ( + "foo.html", + set(), + ), + ( + "bar.html", + {ZimPath("kiwix.org/a/article/bar.html")}, + ), + ], + ) + def test_missing_zim_paths( + self, + original_content_url: str, + expected_missing_zim_paths: set[ZimPath], + ): + http_article_url = HttpUrl("https://kiwix.org/a/article/document.html") + missing_zim_paths = set() + rewriter = ArticleUrlRewriter( + article_url=http_article_url, + existing_zim_paths={ZimPath("kiwix.org/a/article/foo.html")}, + missing_zim_paths=missing_zim_paths, + ) + rewriter(original_content_url, base_href=None, rewrite_all_url=False) + assert missing_zim_paths == expected_missing_zim_paths + + @pytest.mark.parametrize( + "article_url, original_content_url, expected_rewriten_content_url, know_paths, " + "rewrite_all_url", + [ + ( + "https://kiwix.org/a/article/document.html", + "foo.html", + "foo.html", + ["kiwix.org/a/article/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo.html#anchor1", + "foo.html#anchor1", + ["kiwix.org/a/article/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo.html?foo=bar", + "foo.html%3Ffoo%3Dbar", + ["kiwix.org/a/article/foo.html?foo=bar"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo.html?foo=b%24ar", + "foo.html%3Ffoo%3Db%24ar", + ["kiwix.org/a/article/foo.html?foo=b$ar"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo.html?foo=b%3Far", # a query string with an encoded ? char in value + "foo.html%3Ffoo%3Db%3Far", + ["kiwix.org/a/article/foo.html?foo=b?ar"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "fo%o.html", + "fo%25o.html", + ["kiwix.org/a/article/fo%o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foé.html", # URL not matching RFC 3986 (found in invalid HTML doc) + "fo%C3%A9.html", # character is encoded so that URL match RFC 3986 + ["kiwix.org/a/article/foé.html"], # but ZIM path is non-encoded + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "./foo.html", + "foo.html", + ["kiwix.org/a/article/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "../foo.html", + "https://kiwix.org/a/foo.html", # Full URL since not in known URLs + ["kiwix.org/a/article/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "../foo.html", + "../foo.html", # all URLs rewrite activated + ["kiwix.org/a/article/foo.html"], + True, + ), + ( + "https://kiwix.org/a/article/document.html", + "../foo.html", + "../foo.html", + ["kiwix.org/a/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "../bar/foo.html", + "https://kiwix.org/a/bar/foo.html", # Full URL since not in known URLs + ["kiwix.org/a/article/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "../bar/foo.html", + "../bar/foo.html", # all URLs rewrite activated + ["kiwix.org/a/article/foo.html"], + True, + ), + ( + "https://kiwix.org/a/article/document.html", + "../bar/foo.html", + "../bar/foo.html", + ["kiwix.org/a/bar/foo.html"], + False, + ), + ( # we cannot go upper than host, so '../' in excess are removed + "https://kiwix.org/a/article/document.html", + "../../../../../foo.html", + "../../foo.html", + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo?param=value", + "foo%3Fparam%3Dvalue", + ["kiwix.org/a/article/foo?param=value"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo?param=value%2F", + "foo%3Fparam%3Dvalue/", + ["kiwix.org/a/article/foo?param=value/"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo?param=value%2Fend", + "foo%3Fparam%3Dvalue/end", + ["kiwix.org/a/article/foo?param=value/end"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "foo/", + "foo/", + ["kiwix.org/a/article/foo/"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo o.html", + "../../fo%20o.html", + ["kiwix.org/fo o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo+o.html", + "../../fo%2Bo.html", + ["kiwix.org/fo+o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo%2Bo.html", + "../../fo%2Bo.html", + ["kiwix.org/fo+o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/foo.html?param=val+ue", + "../../foo.html%3Fparam%3Dval%20ue", + ["kiwix.org/foo.html?param=val ue"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo~o.html", + "../../fo~o.html", + ["kiwix.org/fo~o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo-o.html", + "../../fo-o.html", + ["kiwix.org/fo-o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo_o.html", + "../../fo_o.html", + ["kiwix.org/fo_o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo%7Eo.html", # must not be encoded / must be decoded (RFC 3986 #2.3) + "../../fo~o.html", + ["kiwix.org/fo~o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo%2Do.html", # must not be encoded / must be decoded (RFC 3986 #2.3) + "../../fo-o.html", + ["kiwix.org/fo-o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/fo%5Fo.html", # must not be encoded / must be decoded (RFC 3986 #2.3) + "../../fo_o.html", + ["kiwix.org/fo_o.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/foo%2Ehtml", # must not be encoded / must be decoded (RFC 3986 #2.3) + "../../foo.html", + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "#anchor1", + "#anchor1", + ["kiwix.org/a/article/document.html"], + False, + ), + ( + "https://kiwix.org/a/article/", + "#anchor1", + "#anchor1", + ["kiwix.org/a/article/"], + False, + ), + ( + "https://kiwix.org/a/article/", + "../article/", + "./", + ["kiwix.org/a/article/"], + False, + ), + ], + ) + def test_relative_url( + self, + article_url: str, + know_paths: list[str], + original_content_url: str, + expected_rewriten_content_url: str, + *, + rewrite_all_url: bool, + ): + http_article_url = HttpUrl(article_url) + rewriter = ArticleUrlRewriter( + article_url=http_article_url, + existing_zim_paths={ZimPath(path) for path in know_paths}, + ) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=rewrite_all_url + ) + == expected_rewriten_content_url + ) + + @pytest.mark.parametrize( + "article_url, original_content_url, expected_rewriten_content_url, know_paths, " + "rewrite_all_url", + [ + ( + "https://kiwix.org/a/article/document.html", + "/foo.html", + "../../foo.html", + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/bar.html", + "https://kiwix.org/bar.html", # Full URL since not in known URLs + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "/bar.html", + "../../bar.html", # all URLs rewrite activated + ["kiwix.org/foo.html"], + True, + ), + ], + ) + def test_absolute_path_url( + self, + article_url: str, + know_paths: list[str], + original_content_url: str, + expected_rewriten_content_url: str, + *, + rewrite_all_url: bool, + ): + http_article_url = HttpUrl(article_url) + rewriter = ArticleUrlRewriter( + article_url=http_article_url, + existing_zim_paths={ZimPath(path) for path in know_paths}, + ) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=rewrite_all_url + ) + == expected_rewriten_content_url + ) + + @pytest.mark.parametrize( + "article_url, original_content_url, expected_rewriten_content_url, know_paths, " + "rewrite_all_url", + [ + ( + "https://kiwix.org/a/article/document.html", + "//kiwix.org/foo.html", + "../../foo.html", + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "//kiwix.org/bar.html", + "https://kiwix.org/bar.html", # Full URL since not in known URLs + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "//kiwix.org/bar.html", + "../../bar.html", # all URLs rewrite activated + ["kiwix.org/foo.html"], + True, + ), + ( + "https://kiwix.org/a/article/document.html", + "//acme.com/foo.html", + "../../../acme.com/foo.html", + ["acme.com/foo.html"], + False, + ), + ( + "http://kiwix.org/a/article/document.html", + "//acme.com/bar.html", + "http://acme.com/bar.html", # Full URL since not in known URLs + ["kiwix.org/foo.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "//acme.com/bar.html", + "../../../acme.com/bar.html", # all URLs rewrite activated + ["kiwix.org/foo.html"], + True, + ), + ( # puny-encoded host is transformed into url-encoded value + "https://kiwix.org/a/article/document.html", + "//xn--exmple-cva.com/a/article/document.html", + "../../../ex%C3%A9mple.com/a/article/document.html", + ["exémple.com/a/article/document.html"], + False, + ), + ( # host who should be puny-encoded ir transformed into url-encoded value + "https://kiwix.org/a/article/document.html", + "//exémple.com/a/article/document.html", + "../../../ex%C3%A9mple.com/a/article/document.html", + ["exémple.com/a/article/document.html"], + False, + ), + ], + ) + def test_absolute_scheme_url( + self, + article_url: str, + know_paths: list[str], + original_content_url: str, + expected_rewriten_content_url: str, + *, + rewrite_all_url: bool, + ): + http_article_url = HttpUrl(article_url) + rewriter = ArticleUrlRewriter( + article_url=http_article_url, + existing_zim_paths={ZimPath(path) for path in know_paths}, + ) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=rewrite_all_url + ) + == expected_rewriten_content_url + ) + + @pytest.mark.parametrize( + "article_url, original_content_url, expected_rewriten_content_url, know_paths, " + "rewrite_all_url", + [ + ( + "https://kiwix.org/a/article/document.html", + "https://foo.org/a/article/document.html", + "../../../foo.org/a/article/document.html", + ["foo.org/a/article/document.html"], + False, + ), + ( + "https://kiwix.org/a/article/document.html", + "http://foo.org/a/article/document.html", + "../../../foo.org/a/article/document.html", + ["foo.org/a/article/document.html"], + False, + ), + ( + "http://kiwix.org/a/article/document.html", + "https://foo.org/a/article/document.html", + "../../../foo.org/a/article/document.html", + ["foo.org/a/article/document.html"], + False, + ), + ( + "http://kiwix.org/a/article/document.html", + "https://user:password@foo.org:8080/a/article/document.html", + "../../../foo.org/a/article/document.html", + ["foo.org/a/article/document.html"], + False, + ), + ( # Full URL since not in known URLs + "https://kiwix.org/a/article/document.html", + "https://foo.org/a/article/document.html", + "https://foo.org/a/article/document.html", + ["kiwix.org/a/article/foo/"], + False, + ), + ( # all URLs rewrite activated + "https://kiwix.org/a/article/document.html", + "https://foo.org/a/article/document.html", + "../../../foo.org/a/article/document.html", + ["kiwix.org/a/article/foo/"], + True, + ), + ( # puny-encoded host is transformed into url-encoded value + "https://kiwix.org/a/article/document.html", + "https://xn--exmple-cva.com/a/article/document.html", + "../../../ex%C3%A9mple.com/a/article/document.html", + ["exémple.com/a/article/document.html"], + False, + ), + ( # host who should be puny-encoded is transformed into url-encoded value + "https://kiwix.org/a/article/document.html", + "https://exémple.com/a/article/document.html", + "../../../ex%C3%A9mple.com/a/article/document.html", + ["exémple.com/a/article/document.html"], + False, + ), + ], + ) + def test_absolute_url( + self, + article_url: str, + know_paths: list[str], + original_content_url: str, + expected_rewriten_content_url: str, + *, + rewrite_all_url: bool, + ): + http_article_url = HttpUrl(article_url) + rewriter = ArticleUrlRewriter( + article_url=http_article_url, + existing_zim_paths={ZimPath(path) for path in know_paths}, + ) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=rewrite_all_url + ) + == expected_rewriten_content_url + ) + + @pytest.mark.parametrize( + "original_content_url, rewrite_all_url", + [ + ("data:0548datacontent", False), + ("blob:exemple.com/url", False), + ("mailto:bob@acme.com", False), + ("tel:+33.1.12.12.23", False), + ("data:0548datacontent", True), + ("blob:exemple.com/url", True), + ("mailto:bob@acme.com", True), + ("tel:+33.1.12.12.23", True), + ], + ) + # other schemes are never rewritten, even when rewrite_all_url is true + def test_no_rewrite_other_schemes( + self, original_content_url: str, *, rewrite_all_url: bool + ): + article_url = HttpUrl("https://kiwix.org/a/article/document.html") + rewriter = ArticleUrlRewriter(article_url=article_url) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=rewrite_all_url + ) + == original_content_url + ) + + @pytest.mark.parametrize( + "original_content_url, know_path, base_href, expected_rewriten_content_url", + [ + pytest.param( + "foo.html", + "kiwix.org/a/article/foo.html", + None, + "foo.html", + id="no_base", + ), + pytest.param( + "foo.html", + "kiwix.org/a/foo.html", + "../", + "../foo.html", + id="parent_base", + ), + pytest.param( + "foo.html", + "kiwix.org/a/bar/foo.html", + "../bar/", + "../bar/foo.html", + id="base_in_another_folder", + ), + pytest.param( + "foo.html", + "www.example.com/foo.html", + "https://www.example.com/", + "../../../www.example.com/foo.html", + id="base_on_absolute_url", + ), + ], + ) + def test_base_href( + self, + original_content_url: str, + know_path: str, + base_href: str, + expected_rewriten_content_url: str, + ): + rewriter = ArticleUrlRewriter( + article_url=HttpUrl("https://kiwix.org/a/article/document.html"), + existing_zim_paths={ZimPath(know_path)}, + ) + assert ( + rewriter(original_content_url, base_href=base_href, rewrite_all_url=False) + == expected_rewriten_content_url + ) + + +class TestHttpUrl: + + @pytest.mark.parametrize( + "http_url", + [("https://bob@acme.com"), ("http://bob@acme.com"), ("hTtPs://bob@acme.com")], + ) + def test_good_http_urls(self, http_url: str): + HttpUrl(http_url) + + @pytest.mark.parametrize( + "http_url", + [("mailto:bob@acme.com"), ("tel:+41.34.34"), ("mailto:https://bob@acme.com")], + ) + def test_bad_http_urls_scheme(self, http_url: str): + with pytest.raises(ValueError, match="Incorrect HttpUrl scheme in value"): + HttpUrl(http_url) + + def test_http_urls_eq(self): + assert HttpUrl("http://bob@acme.com") == HttpUrl("http://bob@acme.com") + + def test_http_urls_hash(self): + assert ( + HttpUrl("http://bob@acme.com").__hash__() + == HttpUrl("http://bob@acme.com").__hash__() + ) + + def test_http_urls_str(self): + assert str(HttpUrl("http://bob@acme.com")) == "HttpUrl(http://bob@acme.com)" + assert f"{HttpUrl("http://bob@acme.com")}" == "HttpUrl(http://bob@acme.com)" + + def test_bad_http_urls_no_host(self): + with pytest.raises(ValueError, match="Unsupported empty hostname in value"): + HttpUrl("https:///bob/index.html") + + def test_bad_http_urls_no_upper(self): + with pytest.raises( + ValueError, match="Unsupported upper-case chars in hostname" + ): + HttpUrl("https://aCmE.COM/index.html") + + +class TestZimPath: + + @pytest.mark.parametrize( + "path", + [ + ("content/index.html"), + ("index.html"), + ], + ) + def test_good_zim_path(self, path: str): + ZimPath(path) + + @pytest.mark.parametrize( + "path", + [ + ("https://bob@acme.com"), + ("http://bob@acme.com"), + ("mailto:bob@acme.com"), + ("tel:+41.34.34"), + ("mailto:https://bob@acme.com"), + ], + ) + def test_bad_zim_path_scheme(self, path: str): + with pytest.raises(ValueError, match="Unexpected scheme in value"): + ZimPath(path) + + @pytest.mark.parametrize( + "path", + [ + ("//acme.com/content/index.html"), + ], + ) + def test_bad_zim_path_hostname(self, path: str): + with pytest.raises(ValueError, match="Unexpected hostname in value"): + ZimPath(path) + + @pytest.mark.parametrize( + "path", + [ + ("//bob@/content/index.html"), + ], + ) + def test_bad_zim_path_user(self, path: str): + with pytest.raises(ValueError, match="Unexpected username in value"): + ZimPath(path) + + @pytest.mark.parametrize( + "path", + [ + ("//:pass@/content/index.html"), + ], + ) + def test_bad_zim_path_pass(self, path: str): + with pytest.raises(ValueError, match="Unexpected password in value"): + ZimPath(path) + + def test_zim_path_eq(self): + assert ZimPath("content/index.html") == ZimPath("content/index.html") + + def test_zim_path_hash(self): + assert ( + ZimPath("content/index.html").__hash__() + == ZimPath("content/index.html").__hash__() + ) + + def test_zim_path_str(self): + assert str(ZimPath("content/index.html")) == "ZimPath(content/index.html)" + assert f"{ZimPath("content/index.html")}" == "ZimPath(content/index.html)" diff --git a/tests/rewriting/utils.py b/tests/rewriting/utils.py new file mode 100644 index 00000000..ee320bd5 --- /dev/null +++ b/tests/rewriting/utils.py @@ -0,0 +1,35 @@ +class ContentForTests: + + def __init__( + self, + input_: str | bytes, + expected: str | bytes | None = None, + article_url: str = "kiwix.org", + ) -> None: + self.input_ = input_ + self.expected = expected if expected is not None else input_ + self.article_url = article_url + + @property + def input_str(self) -> str: + if isinstance(self.input_, str): + return self.input_ + raise ValueError("Input value is not a str.") + + @property + def input_bytes(self) -> bytes: + if isinstance(self.input_, bytes): + return self.input_ + raise ValueError("Input value is not a bytes.") + + @property + def expected_str(self) -> str: + if isinstance(self.expected, str): + return self.expected + raise ValueError("Expected value is not a str.") + + @property + def expected_bytes(self) -> bytes: + if isinstance(self.expected, bytes): + return self.expected + raise ValueError("Expected value is not a bytes.") diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index 2d641691..17425f02 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -425,9 +425,10 @@ def do_GET(self): fpath = tmp_path / "test.zim" try: - with tempfile.TemporaryDirectory() as tmp_dir, Creator( - fpath, "" - ).config_dev_metadata() as creator: + with ( + tempfile.TemporaryDirectory() as tmp_dir, + Creator(fpath, "").config_dev_metadata() as creator, + ): tmp_dir = pathlib.Path(tmp_dir) # noqa: PLW2901 creator.add_item( URLItem( diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..22071e3f --- /dev/null +++ b/yarn.lock @@ -0,0 +1,248 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.25.7.tgz#438f2c524071531d643c6f0188e1e28f130cebc7" + integrity sha512-0xZJFNE5XMpENsgfHYTw8FbX4kv53mFLn2i3XPoq69LyhYSCBJtitaHx9QnsVTrsogI4Z3+HtEfZ2/GFPOtf5g== + dependencies: + "@babel/highlight" "^7.25.7" + picocolors "^1.0.0" + +"@babel/generator@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.25.7.tgz#de86acbeb975a3e11ee92dd52223e6b03b479c56" + integrity sha512-5Dqpl5fyV9pIAD62yK9P7fcA768uVPUyrQmqpqstHWgMma4feF1x/oFysBCVZLY5wJ2GkMUCdsNDnGZrPoR6rA== + dependencies: + "@babel/types" "^7.25.7" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^3.0.2" + +"@babel/helper-string-parser@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.25.7.tgz#d50e8d37b1176207b4fe9acedec386c565a44a54" + integrity sha512-CbkjYdsJNHFk8uqpEkpCvRs3YRp9tY6FmFY7wLMSYuGYkrdUi7r2lc4/wqsvlHoMznX3WJ9IP8giGPq68T/Y6g== + +"@babel/helper-validator-identifier@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.7.tgz#77b7f60c40b15c97df735b38a66ba1d7c3e93da5" + integrity sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg== + +"@babel/highlight@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.25.7.tgz#20383b5f442aa606e7b5e3043b0b1aafe9f37de5" + integrity sha512-iYyACpW3iW8Fw+ZybQK+drQre+ns/tKpXbNESfrhNnPLIklLbXr7MYJ6gPEd0iETGLOK+SxMjVvKb/ffmk+FEw== + dependencies: + "@babel/helper-validator-identifier" "^7.25.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.25.7", "@babel/parser@^7.7.0": + version "7.25.8" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.25.8.tgz#f6aaf38e80c36129460c1657c0762db584c9d5e2" + integrity sha512-HcttkxzdPucv3nNFmfOOMfFf64KgdJVqm1KaCm25dPGMLElo9nsLvXeJECQg8UzPuBGLyTSA0ZzqCtDSzKTEoQ== + dependencies: + "@babel/types" "^7.25.8" + +"@babel/template@^7.25.7": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.7.tgz#27f69ce382855d915b14ab0fe5fb4cbf88fa0769" + integrity sha512-wRwtAgI3bAS+JGU2upWNL9lSlDcRCqD05BZ1n3X2ONLH1WilFP6O1otQjeMK/1g0pvYcXC7b/qVUB1keofjtZA== + dependencies: + "@babel/code-frame" "^7.25.7" + "@babel/parser" "^7.25.7" + "@babel/types" "^7.25.7" + +"@babel/traverse@^7.7.0": + version "7.25.7" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.25.7.tgz#83e367619be1cab8e4f2892ef30ba04c26a40fa8" + integrity sha512-jatJPT1Zjqvh/1FyJs6qAHL+Dzb7sTb+xr7Q+gM1b+1oBsMsQQ4FkVKb6dFlJvLlVssqkRzV05Jzervt9yhnzg== + dependencies: + "@babel/code-frame" "^7.25.7" + "@babel/generator" "^7.25.7" + "@babel/parser" "^7.25.7" + "@babel/template" "^7.25.7" + "@babel/types" "^7.25.7" + debug "^4.3.1" + globals "^11.1.0" + +"@babel/types@^7.25.7", "@babel/types@^7.25.8", "@babel/types@^7.7.0": + version "7.25.8" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.25.8.tgz#5cf6037258e8a9bcad533f4979025140cb9993e1" + integrity sha512-JWtuCu8VQsMladxVz/P4HzHUGCAwpuqacmowgXFs5XjxIgKuNjnLokQzuVjlTvIzODaDmpjT3oxcC48vyk9EWg== + dependencies: + "@babel/helper-string-parser" "^7.25.7" + "@babel/helper-validator-identifier" "^7.25.7" + to-fast-properties "^2.0.0" + +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" + integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== + +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +babel-eslint@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232" + integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg== + dependencies: + "@babel/code-frame" "^7.0.0" + "@babel/parser" "^7.7.0" + "@babel/traverse" "^7.7.0" + "@babel/types" "^7.7.0" + eslint-visitor-keys "^1.0.0" + resolve "^1.12.0" + +chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + +debug@^4.3.1: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== + +eslint-visitor-keys@^1.0.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== + +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== + +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +jsesc@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e" + integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + +picocolors@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + +resolve@^1.12.0: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== From b613fcab2ba4c9656342f91cc3421dbad74c0b74 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 21 Oct 2024 08:48:26 +0000 Subject: [PATCH 2/6] fixup! Move rewriting stuff from warc2zim to zimscraperlib --- .github/workflows/Publish.yaml | 2 +- .pre-commit-config.yaml | 6 +++--- openzim.toml | 9 ++++++++- pyproject.toml | 10 +++++----- rules/rules.yaml | 31 ++++++++++++++++--------------- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml index b8595e79..507a6314 100644 --- a/.github/workflows/Publish.yaml +++ b/.github/workflows/Publish.yaml @@ -1,4 +1,4 @@ -name: Build and publish to PyPI / NPM +name: Build and publish to PyPI on: release: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a3e73b4e..8302a4ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ exclude: ^tests/files # these are raw test files, no need to mess with them repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,11 +12,11 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 + rev: v0.7.0 hooks: - id: ruff - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.368 + rev: v1.1.385 hooks: - id: pyright name: pyright (system) diff --git a/openzim.toml b/openzim.toml index 8db2dc8e..73c13da8 100644 --- a/openzim.toml +++ b/openzim.toml @@ -9,7 +9,14 @@ action="get_file" source="https://cdn.jsdelivr.net/npm/@webrecorder/wombat@3.8.2/dist/wombat.js" target_file="wombat.js" -[files.assets.actions."wombatSetup.js"] # fallback if this script has not been properly built +# wombatSetup.js is supposed to be built locally from files in javascript folder. +# Should someone not have proper skills / tooling / knowledge, or simply install from +# sdist / Github repo directly, without any advanced knowledge of this specificity, the +# configuration below ensures that wombatSetup.js is downloaded from dev.kiwix.org, +# where we have the latest version from `main` branch. wheel contains the wombatSetup.js +# which was built at the same time than the wheel. (reminder: get_file action does not +# overwrite a file which already exists) +[files.assets.actions."wombatSetup.js"] action="get_file" source="https://dev.kiwix.org/zimscraperlib/wombatSetup.js" target_file="wombatSetup.js" diff --git a/pyproject.toml b/pyproject.toml index c0149861..2dacc5c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,20 +52,20 @@ scripts = [ ] lint = [ "black==24.10.0", - "ruff==0.4.9", + "ruff==0.7.0", ] check = [ - "pyright==1.1.368", - "pytest==8.2.2", + "pyright==1.1.385", + "pytest==8.3.3", ] test = [ - "pytest==8.2.2", + "pytest==8.3.3", "pytest-mock==3.14.0", "coverage==7.5.3", ] dev = [ "ipython==8.25.0", - "pre-commit==3.7.1", + "pre-commit==4.0.1", "zimscraperlib[scripts]", "zimscraperlib[lint]", "zimscraperlib[test]", diff --git a/rules/rules.yaml b/rules/rules.yaml index bc5206eb..3403cb97 100644 --- a/rules/rules.yaml +++ b/rules/rules.yaml @@ -1,12 +1,13 @@ # This file comes from an adaptation of rules present in -# https://github.com/webrecorder/wabac.js/blame/main/src/fuzzymatcher.js +# https://github.com/webrecorder/wabac.js/blob/main/src/fuzzymatcher.ts # # Syncing rules is done manually, based on expert knowledge, especially because in # scraperlib we are not really fuzzy matching (searching the best entry among existing # ones) but just rewriting to proper path. # -# This file is in sync with content at commit 879018d5b96962df82340a9a57570bbc0fc67815 -# from June 9, 2024 +# This file is in sync with content at commit +# https://github.com/webrecorder/wabac.js/commit/1c3acfce39e0dc127acf455b04237e9a82062730 +# from October 17, 2024 # # This file should be updated at every release of scraperlib # @@ -27,12 +28,12 @@ fuzzyRules: - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576&key=value fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576 - raw_url: foobargooglevideo.com/videoplaybackandfoo?some=thing&id=1576&key=value - unchanged: true # videoplayback is not followed by `?` + unchanged: true # videoplayback is not followed by `?` - raw_url: foobargoogle_video.com/videoplaybackandfoo?some=thing&id=1576&key=value - unchanged: true # No googlevideo.com in url + unchanged: true # No googlevideo.com in url - name: youtube_video_info pattern: (?:www\.)?youtube(?:-nocookie)?\.com/(get_video_info\?).*(video_id=[^&]+).* - replace : youtube.fuzzy.replayweb.page/\1\2 + replace: youtube.fuzzy.replayweb.page/\1\2 tests: - raw_url: www.youtube.com/get_video_info?video_id=123ah fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah @@ -52,7 +53,7 @@ fuzzyRules: unchanged: true # improper hostname - name: youtube_thumbnails pattern: i\.ytimg\.com\/vi\/(.*?)\/.*?\.(\w*?)(?:\?.*|$) - replace : i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2 + replace: i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2 tests: - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg @@ -64,7 +65,7 @@ fuzzyRules: fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg - name: trim_digits_only pattern: ([^?]+)\?[\d]+$ - replace : \1 + replace: \1 tests: - raw_url: www.example.com/page?1234 fuzzified_url: www.example.com/page @@ -80,7 +81,7 @@ fuzzyRules: unchanged: true - name: youtubei pattern: (?:www\.)?youtube(?:-nocookie)?\.com\/(youtubei\/[^?]+).*(videoId[^&]+).* - replace : youtube.fuzzy.replayweb.page/\1?\2 + replace: youtube.fuzzy.replayweb.page/\1?\2 tests: - raw_url: www.youtube-nocookie.com/youtubei/page/?videoId=123ah fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah @@ -104,7 +105,7 @@ fuzzyRules: unchanged: true - name: youtube_embed pattern: (?:www\.)?youtube(?:-nocookie)?\.com/embed/([^?]+).* - replace : youtube.fuzzy.replayweb.page/embed/\1 + replace: youtube.fuzzy.replayweb.page/embed/\1 tests: - raw_url: www.youtube-nocookie.com/embed/foo fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo @@ -123,7 +124,7 @@ fuzzyRules: - name: vimeo_cdn_fix # custom warc2zim rule intended to fix Vimeo support pattern: .*(?:gcs-vimeo|vod|vod-progressive|vod-adaptive)\.akamaized\.net.*\/(.+?.mp4)\?.*range=(.*?)(?:&.*|$) - replace : vimeo-cdn.fuzzy.replayweb.page/\1?range=\2 + replace: vimeo-cdn.fuzzy.replayweb.page/\1?range=\2 tests: - raw_url: gcs-vimeo.akamaized.net/123.mp4?range=123-456 fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456 @@ -151,7 +152,7 @@ fuzzyRules: unchanged: true - name: vimeo_cdn pattern: .*(?:gcs-vimeo|vod|vod-progressive)\.akamaized\.net.*?\/([\d/]+.mp4)$ - replace : vimeo-cdn.fuzzy.replayweb.page/\1 + replace: vimeo-cdn.fuzzy.replayweb.page/\1 tests: - raw_url: vod.akamaized.net/23.mp4 fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4 @@ -161,7 +162,7 @@ fuzzyRules: fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/01/4423/13/347119375/1398505169.mp4 - name: vimeo_player pattern: .*player.vimeo.com\/(video\/[\d]+)\?.* - replace : vimeo.fuzzy.replayweb.page/\1 + replace: vimeo.fuzzy.replayweb.page/\1 tests: - raw_url: player.vimeo.com/video/1234?foo=bar fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234 @@ -177,7 +178,7 @@ fuzzyRules: unchanged: true - name: i_vimeo_cdn pattern: .*i\.vimeocdn\.com\/(.*)\?.* - replace : i.vimeocdn.fuzzy.replayweb.page/\1 + replace: i.vimeocdn.fuzzy.replayweb.page/\1 tests: - raw_url: i.vimeocdn.com/image/1234?foo=bar fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/image/1234 @@ -185,7 +186,7 @@ fuzzyRules: fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/something/a456 - name: cheatography_com pattern: cheatography\.com\/scripts\/(.*).js.*[?&](v=[^&]+).* - replace : cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2 + replace: cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2 tests: - raw_url: cheatography.com/scripts/useful.min.js?v=2&q=1719438924 fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2 From c0f4e19a0771db761c681e6bbc590d577b79836e Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 21 Oct 2024 09:44:05 +0000 Subject: [PATCH 3/6] Add documentation, especially warc2zim doc about rewriting --- README.md | 2 + docs/functional_architecture.md | 92 +++++++++++++++++++ docs/software_architecture.md | 25 +++++ docs/technical_architecture.md | 52 +++++++++++ openzim.toml | 12 --- pyproject.toml | 7 ++ src/zimscraperlib/rewriting/statics/README.md | 8 ++ 7 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 docs/functional_architecture.md create mode 100644 docs/software_architecture.md create mode 100644 docs/technical_architecture.md create mode 100644 src/zimscraperlib/rewriting/statics/README.md diff --git a/README.md b/README.md index 835832d2..2ea41427 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Example usage: zimscraperlib>=1.1,<1.2 ``` +See [functional architecture](docs/functional_architecture.md), [software architecture](docs/software_architecture.md) and [technical architecture](docs/technical_architecture.md) for more details on scraperlib (not all aspects are covered yet, this is a WIP). + # Dependencies * libmagic diff --git a/docs/functional_architecture.md b/docs/functional_architecture.md new file mode 100644 index 00000000..c3137d8b --- /dev/null +++ b/docs/functional_architecture.md @@ -0,0 +1,92 @@ +# Functional Architecture + +## Enrich libzim functions + +zimscraperlib has primitives to enrich libzim functions with some operations which are known to be shared across scrapers. See `zim` module. + +## Handle videos + +zimscraperlib has primitives to manipulate videos with some operations which are known to be shared across scrapers. See `video` module. + +## Handle pictures + +zimscraperlib has primitives to manipulate pictures with some operations which are known to be shared across scrapers. See `image` module. + +## Store and rewrite mostly unmodified HTML, CSS and JS from online website + +zimscraperlib also contains primitives to rewrite HTML, CSS and JS fetched online, to proper operate within a ZIM without heavy modifications. While originaly developped for warc2zim, some of these primitives are now also used for mindtouch scraper and others might follow, so they are shared in zimscraperlib. See `rewriting` module. + +## ZIM storage + +While storing web resources in a ZIM is mostly straightforward (we just transfer the raw bytes, after some modification for URL rewriting if needed), the decision of the path where the resource will be stored is very important. + +This is purely conventional, even if ZIM specification has to be respected for proper operation in readers. + +This function is responsible to compute the ZIM path where a given web resource is going to be stored. + +While the URL is the only driver of this computation for now, zimscraperlib might have to consider other contextual data in the future. E.g. the resource to serve might by dynamic, depending not only on URL query parameters but also header(s) value(s). + +## Fuzzy rules + +Unfortunately, it is not always possible / desirable to store the resource with a simple transformation. + +A typical situation is that some query parameters are dynamically computed by some Javascript code to include user tracking identifier, current datetime information, ... + +When running again the same javascript code inside the ZIM, the URL will hence be slightly different because context has changed, but the same content needs to be retrieved. + +zimscraperlib hence relies on fuzzy rules to transform/simplify some URLs when computing the ZIM path. + +## URL Rewriting + +zimscraperlib transforms (rewrites) URLs found in documents (HTML, CSS, JS, ...) so that they are usable inside the ZIM. + +### General case + +One simple example is that we might have following code in an HTML document to load an image with an absolute URL: + +``` + +``` + +The URL `https://en.wikipedia.org/wiki/File:Kiwix_logo_v3.svg` has to be transformed to a URL that it is usable inside the ZIM. + +For proper reader operation, openZIM prohibits using absolute URLs, so this has to be a relative URL. This relative URL is hence dependant on the location of the resource currently being rewriten. + +The table below gives some examples of what the rewritten URL is going to be, depending on the URL of the rewritten document. + +| HTML document URL | image URL rewritten for usage inside the ZIM | +|--|--| +| `https://en.wikipedia.org/wiki/Kiwix` | `./File:Kiwix_logo_v3.svg` | +| `https://en.wikipedia.org/wiki` | `./wiki/File:Kiwix_logo_v3.svg` | +| `https://en.wikipedia.org/waka/Kiwix` | `../wiki/File:Kiwix_logo_v3.svg` | +| `https://fr.wikipedia.org/wiki/Kiwix` | `../../en.wikipedia.org/wiki/File:Kiwix_logo_v3.svg` | + +As can be seen on the last line (but this is true for all URLs), this rewriting has to take into account the convention saying at which ZIM path a given web resource will be stored. + +### Dynamic case + +The explanation above more or less assumed that the transformations can be done statically, i.e zimscraperlib can open every known document, find existing URLs and replace them with their counterpart inside the ZIM. + +While this is possible for HTML and CSS documents typically, it is not possible when the URL is dynamically computed. This is typically the case for JS documents, where in the general case the URL is not statically stored inside the JS code but computed on-the-fly by aggregating various strings and values. + +Rewriting these computations is not deemed feasible due to the huge variety of situation which might be encountered. + +A specific function is hence needed to rewrite URL **live in client browser**, intercept any function triggering a web request, transform the URL according to conventions (where we expect the resource to be located in the general case) and fuzzy rules. + +_Spoiler: this is where we will rely on wombat.js from webrecorder team, since this dynamic interception is quite complex and already done quite neatly by them_ + +### Fuzzy rules + +The same fuzzy rules that have been used to compute the ZIM path from a resource URL have to be applied again when rewriting URLs. + +While this is expected to serve mostly for the dynamic case, we still applies them on both side (staticaly and dynamicaly) for coherency. + +## Documents rewriten statically + +For now zimscraperlib rewrites HTML, CSS and JS documents. For CSS and JS, this mainly consists in replacing URLs. For HTML, we also have more specific rewritting necessary (e.g. to handle base href or redirects with meta). + +No domain specific (DS) rules are applied like it is done in wabac.JS because these rules are already applied in Browsertrix Crawler. For the same reason, JSON is not rewritten anymore (URL do not need to be rewritten in JSON because these URLs will be used by JS, intercepted by wombat and dynamically rewritten). + +JSONP callbacks are supposed to be rewritten but this has not been heavily tested. + +Other types of documents are supposed to be either not feasible / not worth it (e.g. URLs inside PDF documents), meaningless (e.g. images, fonts) or planned for later due to limited usage in the wild (e.g. XML). diff --git a/docs/software_architecture.md b/docs/software_architecture.md new file mode 100644 index 00000000..ef5a647d --- /dev/null +++ b/docs/software_architecture.md @@ -0,0 +1,25 @@ +# Software architecture + +## HTML rewriting + +HTML rewriting is purely static (i.e. before resources are written to the ZIM). HTML code is parsed with the [HTML parser from Python standard library](https://docs.python.org/3/library/html.parser.html). + +A small header script is inserted in HTML code to initialize wombat.js which will wrap all JS APIs to dynamically rewrite URLs comming from JS. + +This header script is generated using [Jinja2](https://pypi.org/project/Jinja2/) template since it needs to populate some JS context variables needed by wombat.js operations (original scheme, original url, ...). + +## CSS rewriting + +CSS rewriting is purely static (i.e. before resources are written to the ZIM). CSS code is parsed with the [tinycss2 Python library](https://pypi.org/project/tinycss2/). + +## JS rewriting + +### Static + +Static JS rewriting is simply a matter of pure textual manipulation with regular expressions. No parsing is done at all. + +### Dynamic + +Dynamic JS rewriting is done with [wombat JS library](https://github.com/webrecorder/wombat). The same fuzzy rules that are used for static rewritting are injected into wombat configuration. Code to rewrite URLs is an adapted version of the code used to compute ZIM paths. + +For wombat setup, including the URL rewriting part, we need to pass wombat configuration info. This code is developed in the `javascript` folder. For URL parsing, it relies on the [uri-js library](https://www.npmjs.com/package/uri-js). This javascript code is bundled into a single `wombatSetup.js` file with [rollup bundler](https://rollupjs.org), the same bundler used by webrecorder team to bundle wombat. \ No newline at end of file diff --git a/docs/technical_architecture.md b/docs/technical_architecture.md new file mode 100644 index 00000000..860b2d1a --- /dev/null +++ b/docs/technical_architecture.md @@ -0,0 +1,52 @@ +# Technical architecture + +## Fuzzy rules + +Fuzzy rules are stored in `rules/rules.yaml`. This configuration file is then used by `rules/generateRules.py` to generate Python and JS code. + +Should you update these fuzzy rules, you hence have to: +- regenerate Python and JS files by running `python rules/generateRules.py` +- bundle again Javascript `wombatSetup.js` (see below). + +## Wombat configuration + +Wombat configuration contains some static configuration and the dynamic URL rewriting, including fuzzy rules. + +It is bundled by rollup with `cd javascript && yarn build-prod` and the result is pushed to proper scraper location for inclusion at build time. + +Tests are available and run with `cd javascript && yarn test`. + +## Transformation of URL into ZIM path + +Transforming a URL into a ZIM path has to respect the ZIM specification: path must not be url-encoded (i.e. it must be decoded) and it must be stored as UTF-8. + +WARC record stores the items URL inside a header named "WARC-Target-URI". The value inside this header is encoded, or more exactly it is "exactly what the browser sent at the HTTP level" (see https://github.com/webrecorder/browsertrix-crawler/issues/492 for more details). + +It has been decided (by convention) that we will drop the scheme, the port, the username and password from the URL. Headers are also not considered in this computation. + +Computation of the ZIM path is hence mostly straightforward: +- decode the hostname which is puny-encoded +- decode the path and query parameter which might be url-encoded + +## URL rewriting + +In addition to the computation of the relative path from the current document URL to the URL to rewrite, URL rewriting also consists in computing the proper ZIM path (with same operation as above) and properly encoding it so that the resulting URL respects [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986). Some important stuff has to be noted in this encoding. + +- since the original hostname is now part of the path, it will now be url-encoded +- since the `?` and following query parameters are also part of the path (we do not want readers to drop them like kiwix-serve would do), they are also url-encoded + +Below is an example case of the rewrite operation on an image URL found in an HTML document. + +- Document original URL: `https://kiwix.org/a/article/document.html` +- Document ZIM path: `kiwix.org/a/article/document.html` +- Image original URL: `//xn--exmple-cva.com/a/resource/image.png?foo=bar` +- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo%3Dbar` +- Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar` + +## JS Rewriting + +JS Rewriting is a bit special because rules to apply are different wether we are using "classic" Javascript or "module" Javascript. + +Detection of Javascript modules starts at the HTML level where we have a `