Add webdriver-manager for managing chrome drivers

Pipfile

@@ -10,6 +10,7 @@ pytest = "==6.2.3"
 meilisearch = "==0.16.1"
 requests-iap = "==0.2.0"
 python-keycloak-client = "==0.2.3"
+webdriver-manager = "==3.4.2"
 
 [dev-packages]
 pylint = "==2.8.2"

README.md

@@ -466,7 +466,7 @@ If used, `min_indexed_level` is ignored.
 
 When `js_render` is set to `true`, the scraper will use ChromeDriver. This is needed for pages that are rendered with JavaScript, for example, pages generated with React, Vue, or applications that are running in development mode: `autoreload` `watch`.
 
-After installing ChromeDriver, provide the path to the bin using the following environment variable `CHROMEDRIVER_PATH` (default value is `/usr/bin/chromedriver`).
+After installing ChromeDriver, provide the path to the bin using the following environment variable `CHROMEDRIVER_PATH`. If the variable is not set, the scraper will automatically download and use a compatible version of ChromeDriver.
 
 The default value of `js_render` is `false`.
 

scraper/src/config/browser_handler.py

@@ -1,8 +1,11 @@
 import re
 import os
+import sys
+from distutils.util import strtobool
 from selenium import webdriver
 
 from selenium.webdriver.chrome.options import Options
+from webdriver_manager.chrome import ChromeDriverManager
 from ..custom_downloader_middleware import CustomDownloaderMiddleware
 from ..js_executor import JsExecutor
 
@@ -26,12 +29,42 @@ def init(config_original_content, js_render, user_agent):
             chrome_options.add_argument('--headless')
             chrome_options.add_argument('user-agent={0}'.format(user_agent))
 
-            CHROMEDRIVER_PATH = os.environ.get('CHROMEDRIVER_PATH',
-                                               "/usr/bin/chromedriver")
-            if not os.path.isfile(CHROMEDRIVER_PATH):
-                raise Exception(
-                    "Env CHROMEDRIVER_PATH='{}' is not a path to a file".format(
-                        CHROMEDRIVER_PATH))
+            CHROMEDRIVER_PATH = os.environ.get('CHROMEDRIVER_PATH', '')
+            if not CHROMEDRIVER_PATH or not os.path.isfile(CHROMEDRIVER_PATH):
+                print("Could not find ChromeDriver.")
+                print("Either the Env CHROMEDRIVER_PATH='{}' path is incorrect or "
+                      "ChromeDriver is not installed.".format(CHROMEDRIVER_PATH))
+                print("Do you want to automatically download ChromeDriver?")
+                while(True):
+                    user_input = input("[Y/n]: ")
+                    try:
+                        yes = strtobool(user_input)
+                        break
+                    except ValueError:
+                        print("Please enter a valid input.")
+                        continue
+                if yes:
+                    try:
+                        CHROMEDRIVER_PATH = ChromeDriverManager().install()
+
+                    except Exception as e:
+                        print("Could not download ChromeDriver. "
+                              "Please install ChromeDriver manually.")
+                        print(e)
+                        if sys.platform == "linux" or sys.platform == "darwin":
+                            os.system('read -s -n 1 -p "Press any key to continue..."')
+                        if sys.platform == "win32":
+                            os.system('pause')
+                        sys.exit(1)
+                else:
+                    print("Please install ChromeDriver and set the CHROMEDRIVER_PATH "
+                          "environment variable or remove the render_js option.")
+                    if sys.platform == "linux" or sys.platform == "darwin":
+                        os.system('read -s -n 1 -p "Press any key to continue..."')
+                    if sys.platform == "win32":
+                        os.system('pause')
+                    sys.exit(1)
+
             driver = webdriver.Chrome(
                 CHROMEDRIVER_PATH,
                 options=chrome_options)

tests/config_loader/get_extra_facets_test.py

@@ -19,6 +19,7 @@ def test_extra_facets_should_be_set_from_start_urls_variables_browser(self,
                                                                           monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
 
         c = config({
             "start_urls": [
@@ -43,6 +44,7 @@ def test_extra_facets_should_be_set_from_start_urls_variables_with_two_start_url
             self, monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
 
         c = config({
             "js-render": True,
@@ -74,6 +76,7 @@ def test_extra_facets_should_be_set_from_start_urls_variables_with_multiple_tags
             self, monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
 
         c = config({
             "start_urls": [

tests/config_loader/open_selenium_browser_test.py

@@ -21,6 +21,7 @@ def test_browser_not_needed_by_default(self):
     def test_browser_needed_when_js_render_true(self, monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
         # When
         c = config({
             "js_render": True
@@ -37,6 +38,7 @@ def test_browser_needed_when_config_contains_automatic_tag(self,
                                                                monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
 
         # When
         c = config({

tests/config_loader/start_urls_test.py

@@ -75,6 +75,7 @@ def test_start_urls_should_be_generated_when_there_is_automatic_tagging_browser(
             self, monkeypatch):
         monkeypatch.setattr("selenium.webdriver.chrome",
                             lambda x: MockedInit())
+        monkeypatch.setattr('builtins.input', lambda _: "y")
 
         # When
         c = config({