From d707826500da1c21cc93c2e5ce0f7891270a1663 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 26 May 2022 20:07:32 -0400 Subject: [PATCH] Make performance optimizations for driver downloads --- seleniumbase/console_scripts/sb_install.py | 35 +++-- seleniumbase/core/browser_launcher.py | 153 ++++++++++++++------- seleniumbase/fixtures/constants.py | 4 +- 3 files changed, 132 insertions(+), 60 deletions(-) diff --git a/seleniumbase/console_scripts/sb_install.py b/seleniumbase/console_scripts/sb_install.py index f36c922028d..dd2e338140c 100755 --- a/seleniumbase/console_scripts/sb_install.py +++ b/seleniumbase/console_scripts/sb_install.py @@ -6,7 +6,7 @@ iedriver|operadriver} [OPTIONS] Options: VERSION Specify the version. - (Default chromedriver version = 2.44) + (Default chromedriver = 72.0.3626.69) Use "latest" for the latest version. Use "latest-1" for one less than that. -p OR --path Also copy the driver to /usr/local/bin @@ -44,7 +44,7 @@ urllib3.disable_warnings() DRIVER_DIR = os.path.dirname(os.path.realpath(drivers.__file__)) LOCAL_PATH = "/usr/local/bin/" # On Mac and Linux systems -DEFAULT_CHROMEDRIVER_VERSION = "2.44" # (Specify "latest" to get the latest) +DEFAULT_CHROMEDRIVER_VERSION = "72.0.3626.69" # (Specify "latest" for latest) DEFAULT_GECKODRIVER_VERSION = "v0.31.0" DEFAULT_EDGEDRIVER_VERSION = "101.0.1210.32" # (Looks for LATEST_STABLE first) DEFAULT_OPERADRIVER_VERSION = "v.96.0.4664.45" @@ -61,7 +61,7 @@ def invalid_run_command(): exp += " iedriver, operadriver)\n" exp += " Options:\n" exp += " VERSION Specify the version.\n" - exp += " (Default chromedriver version = 2.44)\n" + exp += " (Default chromedriver = 72.0.3626.69.\n" exp += ' Use "latest" for the latest version.\n' exp += " For chromedriver, you can also use\n" exp += " the major version integer\n" @@ -108,14 +108,27 @@ def requests_get(url): def main(override=None): - if override == "chromedriver": - sys.argv = ["seleniumbase", "get", "chromedriver"] - elif override == "edgedriver": - sys.argv = ["seleniumbase", "get", "edgedriver"] - elif override == "geckodriver": - sys.argv = ["seleniumbase", "get", "geckodriver"] - elif override == "iedriver": - sys.argv = ["seleniumbase", "get", "iedriver"] + if override: + if override == "chromedriver": + sys.argv = ["seleniumbase", "get", "chromedriver"] + elif override.startswith("chromedriver "): + extra = override.split("chromedriver ")[1] + sys.argv = ["seleniumbase", "get", "chromedriver", extra] + elif override == "edgedriver": + sys.argv = ["seleniumbase", "get", "edgedriver"] + elif override.startswith("edgedriver "): + extra = override.split("edgedriver ")[1] + sys.argv = ["seleniumbase", "get", "edgedriver", extra] + elif override == "geckodriver": + sys.argv = ["seleniumbase", "get", "geckodriver"] + elif override.startswith("geckodriver "): + extra = override.split("geckodriver ")[1] + sys.argv = ["seleniumbase", "get", "geckodriver", extra] + elif override == "iedriver": + sys.argv = ["seleniumbase", "get", "iedriver"] + elif override.startswith("iedriver "): + extra = override.split("iedriver ")[1] + sys.argv = ["seleniumbase", "get", "iedriver", extra] num_args = len(sys.argv) if ( diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 76c080594fd..0b2a6f33223 100755 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -116,11 +116,18 @@ def is_headless_iedriver_on_path(): return os.path.exists(LOCAL_HEADLESS_IEDRIVER) -def _repair_chromedriver(chrome_options, headless_options): +def _repair_chromedriver(chrome_options, headless_options, mcv=None): import subprocess + if mcv: + subprocess.check_call( + "sbase install chromedriver %s" % mcv, shell=True + ) + return driver = None - subprocess.check_call("sbase install chromedriver 2.44", shell=True) + subprocess.check_call( + "sbase install chromedriver 72.0.3626.69", shell=True + ) try: if selenium4: service = ChromeService(executable_path=LOCAL_CHROMEDRIVER) @@ -149,7 +156,10 @@ def _repair_chromedriver(chrome_options, headless_options): chromedriver_ver = chrome_dict["chromedriverVersion"] chromedriver_ver = chromedriver_ver.split(" ")[0] major_chromedriver_ver = chromedriver_ver.split(".")[0] - if major_chromedriver_ver != major_chrome_ver: + if ( + major_chromedriver_ver != major_chrome_ver + and int(major_chrome_ver) >= 73 + ): subprocess.check_call( "sbase install chromedriver %s" % major_chrome_ver, shell=True ) @@ -169,12 +179,12 @@ def _repair_edgedriver(edge_version): return -def _mark_chromedriver_repaired(): +def _mark_driver_repaired(): import codecs abs_path = os.path.abspath(".") - chromedriver_repaired_lock = constants.MultiBrowser.CHROMEDRIVER_REPAIRED - file_path = os.path.join(abs_path, chromedriver_repaired_lock) + driver_repaired_lock = constants.MultiBrowser.DRIVER_REPAIRED + file_path = os.path.join(abs_path, driver_repaired_lock) downloads_folder = download_helper.get_downloads_folder() if not os.path.exists(downloads_folder): os.makedirs(downloads_folder) @@ -183,10 +193,10 @@ def _mark_chromedriver_repaired(): out_file.close() -def _was_chromedriver_repaired(): +def _was_driver_repaired(): abs_path = os.path.abspath(".") - chromedriver_repaired_lock = constants.MultiBrowser.CHROMEDRIVER_REPAIRED - file_path = os.path.join(abs_path, chromedriver_repaired_lock) + driver_repaired_lock = constants.MultiBrowser.DRIVER_REPAIRED + file_path = os.path.join(abs_path, driver_repaired_lock) return os.path.exists(file_path) @@ -1400,11 +1410,11 @@ def get_local_driver( " executable: %s" % e ) elif not is_geckodriver_on_path(): + from seleniumbase.console_scripts import sb_install + args = " ".join(sys.argv) if not ("-n" in sys.argv or " -n=" in args or args == "-c"): # (Not multithreaded) - from seleniumbase.console_scripts import sb_install - sys_args = sys.argv # Save a copy of current sys args print("\nWarning: geckodriver not found. Getting it now:") try: @@ -1412,6 +1422,21 @@ def get_local_driver( except Exception as e: print("\nWarning: Could not install geckodriver: %s" % e) sys.argv = sys_args # Put back the original sys args + else: + import fasteners + + geckodriver_fixing_lock = fasteners.InterProcessLock( + constants.MultiBrowser.DRIVER_FIXING_LOCK + ) + with geckodriver_fixing_lock: + if not is_geckodriver_on_path(): + sys_args = sys.argv # Save a copy of sys args + print( + "\nWarning: geckodriver not found. " + "Getting it now:" + ) + sb_install.main(override="geckodriver") + sys.argv = sys_args # Put back original sys args warnings.simplefilter("ignore", category=DeprecationWarning) if "linux" in PLATFORM: from selenium.webdriver.common.desired_capabilities import ( @@ -1489,11 +1514,11 @@ def get_local_driver( "\nWarning: Could not make IEDriver executable: %s" % e ) elif not is_iedriver_on_path(): + from seleniumbase.console_scripts import sb_install + args = " ".join(sys.argv) if not ("-n" in sys.argv or " -n=" in args or args == "-c"): # (Not multithreaded) - from seleniumbase.console_scripts import sb_install - sys_args = sys.argv # Save a copy of current sys args print("\nWarning: IEDriver not found. Getting it now:") sb_install.main(override="iedriver") @@ -1507,11 +1532,11 @@ def get_local_driver( % e ) elif not is_headless_iedriver_on_path(): + from seleniumbase.console_scripts import sb_install + args = " ".join(sys.argv) if not ("-n" in sys.argv or " -n=" in args or args == "-c"): # (Not multithreaded) - from seleniumbase.console_scripts import sb_install - sys_args = sys.argv # Save a copy of current sys args print("\nWarning: HeadlessIEDriver not found. Getting it now:") sb_install.main(override="iedriver") @@ -1553,15 +1578,30 @@ def get_local_driver( " executable: %s" % e ) elif not is_edgedriver_on_path(): + from seleniumbase.console_scripts import sb_install + args = " ".join(sys.argv) if not ("-n" in sys.argv or " -n=" in args or args == "-c"): # (Not multithreaded) - from seleniumbase.console_scripts import sb_install - sys_args = sys.argv # Save a copy of current sys args print("\nWarning: msedgedriver not found. Getting it now:") sb_install.main(override="edgedriver") sys.argv = sys_args # Put back the original sys args + else: + import fasteners + + edgedriver_fixing_lock = fasteners.InterProcessLock( + constants.MultiBrowser.DRIVER_FIXING_LOCK + ) + with edgedriver_fixing_lock: + if not is_edgedriver_on_path(): + sys_args = sys.argv # Save a copy of sys args + print( + "\nWarning: msedgedriver not found. " + "Getting it now:" + ) + sb_install.main(override="edgedriver") + sys.argv = sys_args # Put back original sys args # For Microsoft Edge (Chromium) version 80 or higher if selenium4: @@ -1732,16 +1772,16 @@ def get_local_driver( import fasteners edgedriver_fixing_lock = fasteners.InterProcessLock( - constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK + constants.MultiBrowser.DRIVER_FIXING_LOCK ) with edgedriver_fixing_lock: - if not _was_chromedriver_repaired(): # Works for Edge + if not _was_driver_repaired(): _repair_edgedriver(edge_version) - _mark_chromedriver_repaired() # Works for Edge + _mark_driver_repaired() else: - if not _was_chromedriver_repaired(): # Works for Edge + if not _was_driver_repaired(): _repair_edgedriver(edge_version) - _mark_chromedriver_repaired() # Works for Edge + _mark_driver_repaired() service = EdgeService( executable_path=LOCAL_EDGEDRIVER, log_path=os.path.devnull ) @@ -1779,16 +1819,16 @@ def get_local_driver( import fasteners edgedriver_fixing_lock = fasteners.InterProcessLock( - constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK + constants.MultiBrowser.DRIVER_FIXING_LOCK ) with edgedriver_fixing_lock: - if not _was_chromedriver_repaired(): # Works for Edge + if not _was_driver_repaired(): _repair_edgedriver(edge_version) - _mark_chromedriver_repaired() # Works for Edge + _mark_driver_repaired() else: - if not _was_chromedriver_repaired(): # Works for Edge + if not _was_driver_repaired(): _repair_edgedriver(edge_version) - _mark_chromedriver_repaired() # Works for Edge + _mark_driver_repaired() driver = Edge( executable_path=LOCAL_EDGEDRIVER, service_log_path=os.path.devnull, @@ -1912,21 +1952,20 @@ def get_local_driver( " executable: %s" % e ) elif not is_chromedriver_on_path(): + from seleniumbase.console_scripts import sb_install + args = " ".join(sys.argv) if not ("-n" in sys.argv or " -n=" in args or args == "-c"): # (Not multithreaded) - from seleniumbase.console_scripts import sb_install - sys_args = sys.argv # Save a copy of current sys args print("\nWarning: chromedriver not found. Getting it now:") - sb_install.main(override="chromedriver") + sb_install.main(override="chromedriver latest") sys.argv = sys_args # Put back the original sys args else: import fasteners - from seleniumbase.console_scripts import sb_install chromedriver_fixing_lock = fasteners.InterProcessLock( - constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK + constants.MultiBrowser.DRIVER_FIXING_LOCK ) with chromedriver_fixing_lock: if not is_chromedriver_on_path(): @@ -1935,7 +1974,7 @@ def get_local_driver( "\nWarning: chromedriver not found. " "Getting it now:" ) - sb_install.main(override="chromedriver") + sb_install.main(override="chromedriver latest") sys.argv = sys_args # Put back original sys args if not headless or "linux" not in PLATFORM: try: @@ -1976,6 +2015,16 @@ def get_local_driver( raise Exception(e.msg) # Not an obvious fix. Raise. else: pass # Try upgrading ChromeDriver to match Chrome. + mcv = None # Major Chrome Version + if "Current browser version is " in e.msg: + line = e.msg.split("Current browser version is ")[1] + browser_version = line.split(" ")[0] + major_chrome_version = browser_version.split(".")[0] + if ( + major_chrome_version.isnumeric() + and int(major_chrome_version) >= 86 + ): + mcv = major_chrome_version headless = True headless_options = _set_chrome_options( browser_name, @@ -2018,20 +2067,20 @@ def get_local_driver( import fasteners chromedriver_fixing_lock = fasteners.InterProcessLock( - constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK + constants.MultiBrowser.DRIVER_FIXING_LOCK ) with chromedriver_fixing_lock: - if not _was_chromedriver_repaired(): + if not _was_driver_repaired(): _repair_chromedriver( - chrome_options, headless_options + chrome_options, headless_options, mcv ) - _mark_chromedriver_repaired() + _mark_driver_repaired() else: - if not _was_chromedriver_repaired(): + if not _was_driver_repaired(): _repair_chromedriver( - chrome_options, headless_options + chrome_options, headless_options, mcv ) - _mark_chromedriver_repaired() + _mark_driver_repaired() if os.path.exists(LOCAL_CHROMEDRIVER): if selenium4: service = ChromeService( @@ -2058,32 +2107,42 @@ def get_local_driver( auto_upgrade_chromedriver = True elif "Chrome version must be between" in e.msg: auto_upgrade_chromedriver = True + mcv = None # Major Chrome Version + if "Current browser version is " in e.msg: + line = e.msg.split("Current browser version is ")[1] + browser_version = line.split(" ")[0] + major_chrome_version = browser_version.split(".")[0] + if ( + major_chrome_version.isnumeric() + and int(major_chrome_version) >= 86 + ): + mcv = major_chrome_version if auto_upgrade_chromedriver: args = " ".join(sys.argv) if "-n" in sys.argv or " -n=" in args or args == "-c": import fasteners chromedr_fixing_lock = fasteners.InterProcessLock( - constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK + constants.MultiBrowser.DRIVER_FIXING_LOCK ) with chromedr_fixing_lock: - if not _was_chromedriver_repaired(): + if not _was_driver_repaired(): try: _repair_chromedriver( - chrome_options, chrome_options + chrome_options, chrome_options, mcv ) - _mark_chromedriver_repaired() + _mark_driver_repaired() except Exception: pass else: - if not _was_chromedriver_repaired(): + if not _was_driver_repaired(): try: _repair_chromedriver( - chrome_options, chrome_options + chrome_options, chrome_options, mcv ) except Exception: pass - _mark_chromedriver_repaired() + _mark_driver_repaired() try: return webdriver.Chrome(options=chrome_options) except Exception: diff --git a/seleniumbase/fixtures/constants.py b/seleniumbase/fixtures/constants.py index 5ec8af1348f..dc8274dd24e 100755 --- a/seleniumbase/fixtures/constants.py +++ b/seleniumbase/fixtures/constants.py @@ -79,8 +79,8 @@ class SideBySide: class MultiBrowser: - CHROMEDRIVER_FIXING_LOCK = Files.DOWNLOADS_FOLDER + "/driver_fixing.lock" - CHROMEDRIVER_REPAIRED = Files.DOWNLOADS_FOLDER + "/driver_fixed.lock" + DRIVER_FIXING_LOCK = Files.DOWNLOADS_FOLDER + "/driver_fixing.lock" + DRIVER_REPAIRED = Files.DOWNLOADS_FOLDER + "/driver_fixed.lock" class SavedCookies: