From 7e2027169c66175d970baf0f81b10c79ed2bb326 Mon Sep 17 00:00:00 2001
From: Brendan Kenny <bckenny@gmail.com>
Date: Mon, 10 Jan 2022 13:31:41 -0600
Subject: [PATCH] deps: update robots-parser to latest

---
 build/build-bundle.js                      |  1 -
 lighthouse-core/audits/seo/is-crawlable.js |  3 ++-
 package.json                               |  2 +-
 types/robots-parser/index.d.ts             | 20 --------------------
 yarn.lock                                  |  8 ++++----
 5 files changed, 7 insertions(+), 27 deletions(-)
 delete mode 100644 types/robots-parser/index.d.ts

diff --git a/build/build-bundle.js b/build/build-bundle.js
index fecc7f6bbcaf..2324a4bb26cf 100644
--- a/build/build-bundle.js
+++ b/build/build-bundle.js
@@ -155,7 +155,6 @@ async function build(entryPath, distPath, opts = {minify: true}) {
         // that's all that's needed, so make a mini-polyfill.
         // @see https://github.com/GoogleChrome/lighthouse/issues/5273
         // TODO: remove when not needed for pubads (https://github.com/googleads/publisher-ads-lighthouse-plugin/pull/325)
-        // and robots-parser (https://github.com/samclarke/robots-parser/pull/23)
         'url': 'export const URL = globalThis.URL;',
       }),
       rollupPlugins.json(),
diff --git a/lighthouse-core/audits/seo/is-crawlable.js b/lighthouse-core/audits/seo/is-crawlable.js
index b76ab539a3f6..f356589e9b20 100644
--- a/lighthouse-core/audits/seo/is-crawlable.js
+++ b/lighthouse-core/audits/seo/is-crawlable.js
@@ -6,7 +6,8 @@
 'use strict';
 
 const Audit = require('../audit.js');
-const robotsParser = require('robots-parser');
+// TODO(esmodules): cast can be removed when this switches to import.
+const robotsParser = /** @type {typeof import('robots-parser').default} */ (/** @type {unknown} */(require('robots-parser'))); // eslint-disable-line max-len
 const URL = require('../../lib/url-shim.js');
 const MainResource = require('../../computed/main-resource.js');
 const BLOCKLIST = new Set([
diff --git a/package.json b/package.json
index be9c55df326b..ac70851e24e6 100644
--- a/package.json
+++ b/package.json
@@ -206,7 +206,7 @@
     "parse-cache-control": "1.0.1",
     "ps-list": "^8.0.0",
     "raven": "^2.2.1",
-    "robots-parser": "^2.3.0",
+    "robots-parser": "^3.0.0",
     "semver": "^5.3.0",
     "speedline-core": "^1.4.3",
     "third-party-web": "^0.12.7",
diff --git a/types/robots-parser/index.d.ts b/types/robots-parser/index.d.ts
deleted file mode 100644
index 9d916abc8d43..000000000000
--- a/types/robots-parser/index.d.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
- */
-
-declare module 'robots-parser' {
-  interface Robots {
-    isAllowed(url: string, ua?: string): boolean | undefined;
-    isDisallowed(url: string, ua?: string): boolean | undefined;
-    getMatchingLineNumber(url: string, ua?: string): number | undefined;
-    getCrawlDelay(ua?: string): number | undefined;
-    getSitemaps(): Array<string>;
-    getPreferredHost(): string | null;
-  }
-
-  function RobotsParser(url: string, robotsTxt: string): Robots;
-
-  export = RobotsParser;
-}
diff --git a/yarn.lock b/yarn.lock
index 6dc8827dcff7..14828b8eaa53 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7153,10 +7153,10 @@ rimraf@^2.7.1:
   dependencies:
     glob "^7.1.3"
 
-robots-parser@^2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/robots-parser/-/robots-parser-2.3.0.tgz#d79e86e26e13fa0a806adbc37f4cf1b96aebc8c3"
-  integrity sha512-RvuCITckrHM9k8DxCCU9rqWpuuKRfVX9iHG751dC3/EdERxp9gJATxYYdYOT3L0T+TAT4+27lENisk/VbHm47A==
+robots-parser@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/robots-parser/-/robots-parser-3.0.0.tgz#66af89306302ecd004455f2f24298310d0966631"
+  integrity sha512-6xkze3WRdneibICBAzMKcXyTKQw5shA3GbwoEJy7RSvxpZNGF0GMuYKE1T0VMP4fwx/fQs0n0mtriOqRtk5L1w==
 
 rollup-plugin-node-resolve@^5.2.0:
   version "5.2.0"