diff --git a/build/build-bundle.js b/build/build-bundle.js index fecc7f6bbcaf..2324a4bb26cf 100644 --- a/build/build-bundle.js +++ b/build/build-bundle.js @@ -155,7 +155,6 @@ async function build(entryPath, distPath, opts = {minify: true}) { // that's all that's needed, so make a mini-polyfill. // @see https://github.com/GoogleChrome/lighthouse/issues/5273 // TODO: remove when not needed for pubads (https://github.com/googleads/publisher-ads-lighthouse-plugin/pull/325) - // and robots-parser (https://github.com/samclarke/robots-parser/pull/23) 'url': 'export const URL = globalThis.URL;', }), rollupPlugins.json(), diff --git a/lighthouse-core/audits/seo/is-crawlable.js b/lighthouse-core/audits/seo/is-crawlable.js index b76ab539a3f6..f356589e9b20 100644 --- a/lighthouse-core/audits/seo/is-crawlable.js +++ b/lighthouse-core/audits/seo/is-crawlable.js @@ -6,7 +6,8 @@ 'use strict'; const Audit = require('../audit.js'); -const robotsParser = require('robots-parser'); +// TODO(esmodules): cast can be removed when this switches to import. +const robotsParser = /** @type {typeof import('robots-parser').default} */ (/** @type {unknown} */(require('robots-parser'))); // eslint-disable-line max-len const URL = require('../../lib/url-shim.js'); const MainResource = require('../../computed/main-resource.js'); const BLOCKLIST = new Set([ diff --git a/package.json b/package.json index be9c55df326b..ac70851e24e6 100644 --- a/package.json +++ b/package.json @@ -206,7 +206,7 @@ "parse-cache-control": "1.0.1", "ps-list": "^8.0.0", "raven": "^2.2.1", - "robots-parser": "^2.3.0", + "robots-parser": "^3.0.0", "semver": "^5.3.0", "speedline-core": "^1.4.3", "third-party-web": "^0.12.7", diff --git a/types/robots-parser/index.d.ts b/types/robots-parser/index.d.ts deleted file mode 100644 index 9d916abc8d43..000000000000 --- a/types/robots-parser/index.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ - -declare module 'robots-parser' { - interface Robots { - isAllowed(url: string, ua?: string): boolean | undefined; - isDisallowed(url: string, ua?: string): boolean | undefined; - getMatchingLineNumber(url: string, ua?: string): number | undefined; - getCrawlDelay(ua?: string): number | undefined; - getSitemaps(): Array; - getPreferredHost(): string | null; - } - - function RobotsParser(url: string, robotsTxt: string): Robots; - - export = RobotsParser; -} diff --git a/yarn.lock b/yarn.lock index 6dc8827dcff7..14828b8eaa53 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7153,10 +7153,10 @@ rimraf@^2.7.1: dependencies: glob "^7.1.3" -robots-parser@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/robots-parser/-/robots-parser-2.3.0.tgz#d79e86e26e13fa0a806adbc37f4cf1b96aebc8c3" - integrity sha512-RvuCITckrHM9k8DxCCU9rqWpuuKRfVX9iHG751dC3/EdERxp9gJATxYYdYOT3L0T+TAT4+27lENisk/VbHm47A== +robots-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/robots-parser/-/robots-parser-3.0.0.tgz#66af89306302ecd004455f2f24298310d0966631" + integrity sha512-6xkze3WRdneibICBAzMKcXyTKQw5shA3GbwoEJy7RSvxpZNGF0GMuYKE1T0VMP4fwx/fQs0n0mtriOqRtk5L1w== rollup-plugin-node-resolve@^5.2.0: version "5.2.0"