Skip to content

Commit

Permalink
Make performance optimizations for driver downloads
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdmintz
mdmintz committed May 27, 2022
1 parent 146c374 commit d707826
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 60 deletions.
35 changes: 24 additions & 11 deletions seleniumbase/console_scripts/sb_install.py
View file
Expand Up @@ -6,7 +6,7 @@
iedriver|operadriver} [OPTIONS]
Options:
VERSION Specify the version.
(Default chromedriver version = 2.44)
(Default chromedriver = 72.0.3626.69)
Use "latest" for the latest version.
Use "latest-1" for one less than that.
-p OR --path Also copy the driver to /usr/local/bin
Expand Down Expand Up @@ -44,7 +44,7 @@
urllib3.disable_warnings()
DRIVER_DIR = os.path.dirname(os.path.realpath(drivers.__file__))
LOCAL_PATH = "/usr/local/bin/" # On Mac and Linux systems
DEFAULT_CHROMEDRIVER_VERSION = "2.44" # (Specify "latest" to get the latest)
DEFAULT_CHROMEDRIVER_VERSION = "72.0.3626.69" # (Specify "latest" for latest)
DEFAULT_GECKODRIVER_VERSION = "v0.31.0"
DEFAULT_EDGEDRIVER_VERSION = "101.0.1210.32" # (Looks for LATEST_STABLE first)
DEFAULT_OPERADRIVER_VERSION = "v.96.0.4664.45"
Expand All @@ -61,7 +61,7 @@ def invalid_run_command():
exp += " iedriver, operadriver)\n"
exp += " Options:\n"
exp += " VERSION Specify the version.\n"
exp += " (Default chromedriver version = 2.44)\n"
exp += " (Default chromedriver = 72.0.3626.69.\n"
exp += ' Use "latest" for the latest version.\n'
exp += " For chromedriver, you can also use\n"
exp += " the major version integer\n"
Expand Down Expand Up @@ -108,14 +108,27 @@ def requests_get(url):


def main(override=None):
if override == "chromedriver":
sys.argv = ["seleniumbase", "get", "chromedriver"]
elif override == "edgedriver":
sys.argv = ["seleniumbase", "get", "edgedriver"]
elif override == "geckodriver":
sys.argv = ["seleniumbase", "get", "geckodriver"]
elif override == "iedriver":
sys.argv = ["seleniumbase", "get", "iedriver"]
if override:
if override == "chromedriver":
sys.argv = ["seleniumbase", "get", "chromedriver"]
elif override.startswith("chromedriver "):
extra = override.split("chromedriver ")[1]
sys.argv = ["seleniumbase", "get", "chromedriver", extra]
elif override == "edgedriver":
sys.argv = ["seleniumbase", "get", "edgedriver"]
elif override.startswith("edgedriver "):
extra = override.split("edgedriver ")[1]
sys.argv = ["seleniumbase", "get", "edgedriver", extra]
elif override == "geckodriver":
sys.argv = ["seleniumbase", "get", "geckodriver"]
elif override.startswith("geckodriver "):
extra = override.split("geckodriver ")[1]
sys.argv = ["seleniumbase", "get", "geckodriver", extra]
elif override == "iedriver":
sys.argv = ["seleniumbase", "get", "iedriver"]
elif override.startswith("iedriver "):
extra = override.split("iedriver ")[1]
sys.argv = ["seleniumbase", "get", "iedriver", extra]

num_args = len(sys.argv)
if (
Expand Down
153 changes: 106 additions & 47 deletions seleniumbase/core/browser_launcher.py
View file
Expand Up @@ -116,11 +116,18 @@ def is_headless_iedriver_on_path():
return os.path.exists(LOCAL_HEADLESS_IEDRIVER)


def _repair_chromedriver(chrome_options, headless_options):
def _repair_chromedriver(chrome_options, headless_options, mcv=None):
import subprocess

if mcv:
subprocess.check_call(
"sbase install chromedriver %s" % mcv, shell=True
)
return
driver = None
subprocess.check_call("sbase install chromedriver 2.44", shell=True)
subprocess.check_call(
"sbase install chromedriver 72.0.3626.69", shell=True
)
try:
if selenium4:
service = ChromeService(executable_path=LOCAL_CHROMEDRIVER)
Expand Down Expand Up @@ -149,7 +156,10 @@ def _repair_chromedriver(chrome_options, headless_options):
chromedriver_ver = chrome_dict["chromedriverVersion"]
chromedriver_ver = chromedriver_ver.split(" ")[0]
major_chromedriver_ver = chromedriver_ver.split(".")[0]
if major_chromedriver_ver != major_chrome_ver:
if (
major_chromedriver_ver != major_chrome_ver
and int(major_chrome_ver) >= 73
):
subprocess.check_call(
"sbase install chromedriver %s" % major_chrome_ver, shell=True
)
Expand All @@ -169,12 +179,12 @@ def _repair_edgedriver(edge_version):
return


def _mark_chromedriver_repaired():
def _mark_driver_repaired():
import codecs

abs_path = os.path.abspath(".")
chromedriver_repaired_lock = constants.MultiBrowser.CHROMEDRIVER_REPAIRED
file_path = os.path.join(abs_path, chromedriver_repaired_lock)
driver_repaired_lock = constants.MultiBrowser.DRIVER_REPAIRED
file_path = os.path.join(abs_path, driver_repaired_lock)
downloads_folder = download_helper.get_downloads_folder()
if not os.path.exists(downloads_folder):
os.makedirs(downloads_folder)
Expand All @@ -183,10 +193,10 @@ def _mark_chromedriver_repaired():
out_file.close()


def _was_chromedriver_repaired():
def _was_driver_repaired():
abs_path = os.path.abspath(".")
chromedriver_repaired_lock = constants.MultiBrowser.CHROMEDRIVER_REPAIRED
file_path = os.path.join(abs_path, chromedriver_repaired_lock)
driver_repaired_lock = constants.MultiBrowser.DRIVER_REPAIRED
file_path = os.path.join(abs_path, driver_repaired_lock)
return os.path.exists(file_path)


Expand Down Expand Up @@ -1400,18 +1410,33 @@ def get_local_driver(
" executable: %s" % e
)
elif not is_geckodriver_on_path():
from seleniumbase.console_scripts import sb_install

args = " ".join(sys.argv)
if not ("-n" in sys.argv or " -n=" in args or args == "-c"):
# (Not multithreaded)
from seleniumbase.console_scripts import sb_install

sys_args = sys.argv # Save a copy of current sys args
print("\nWarning: geckodriver not found. Getting it now:")
try:
sb_install.main(override="geckodriver")
except Exception as e:
print("\nWarning: Could not install geckodriver: %s" % e)
sys.argv = sys_args # Put back the original sys args
else:
import fasteners

geckodriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with geckodriver_fixing_lock:
if not is_geckodriver_on_path():
sys_args = sys.argv # Save a copy of sys args
print(
"\nWarning: geckodriver not found. "
"Getting it now:"
)
sb_install.main(override="geckodriver")
sys.argv = sys_args # Put back original sys args
warnings.simplefilter("ignore", category=DeprecationWarning)
if "linux" in PLATFORM:
from selenium.webdriver.common.desired_capabilities import (
Expand Down Expand Up @@ -1489,11 +1514,11 @@ def get_local_driver(
"\nWarning: Could not make IEDriver executable: %s" % e
)
elif not is_iedriver_on_path():
from seleniumbase.console_scripts import sb_install

args = " ".join(sys.argv)
if not ("-n" in sys.argv or " -n=" in args or args == "-c"):
# (Not multithreaded)
from seleniumbase.console_scripts import sb_install

sys_args = sys.argv # Save a copy of current sys args
print("\nWarning: IEDriver not found. Getting it now:")
sb_install.main(override="iedriver")
Expand All @@ -1507,11 +1532,11 @@ def get_local_driver(
% e
)
elif not is_headless_iedriver_on_path():
from seleniumbase.console_scripts import sb_install

args = " ".join(sys.argv)
if not ("-n" in sys.argv or " -n=" in args or args == "-c"):
# (Not multithreaded)
from seleniumbase.console_scripts import sb_install

sys_args = sys.argv # Save a copy of current sys args
print("\nWarning: HeadlessIEDriver not found. Getting it now:")
sb_install.main(override="iedriver")
Expand Down Expand Up @@ -1553,15 +1578,30 @@ def get_local_driver(
" executable: %s" % e
)
elif not is_edgedriver_on_path():
from seleniumbase.console_scripts import sb_install

args = " ".join(sys.argv)
if not ("-n" in sys.argv or " -n=" in args or args == "-c"):
# (Not multithreaded)
from seleniumbase.console_scripts import sb_install

sys_args = sys.argv # Save a copy of current sys args
print("\nWarning: msedgedriver not found. Getting it now:")
sb_install.main(override="edgedriver")
sys.argv = sys_args # Put back the original sys args
else:
import fasteners

edgedriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with edgedriver_fixing_lock:
if not is_edgedriver_on_path():
sys_args = sys.argv # Save a copy of sys args
print(
"\nWarning: msedgedriver not found. "
"Getting it now:"
)
sb_install.main(override="edgedriver")
sys.argv = sys_args # Put back original sys args

# For Microsoft Edge (Chromium) version 80 or higher
if selenium4:
Expand Down Expand Up @@ -1732,16 +1772,16 @@ def get_local_driver(
import fasteners

edgedriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with edgedriver_fixing_lock:
if not _was_chromedriver_repaired(): # Works for Edge
if not _was_driver_repaired():
_repair_edgedriver(edge_version)
_mark_chromedriver_repaired() # Works for Edge
_mark_driver_repaired()
else:
if not _was_chromedriver_repaired(): # Works for Edge
if not _was_driver_repaired():
_repair_edgedriver(edge_version)
_mark_chromedriver_repaired() # Works for Edge
_mark_driver_repaired()
service = EdgeService(
executable_path=LOCAL_EDGEDRIVER, log_path=os.path.devnull
)
Expand Down Expand Up @@ -1779,16 +1819,16 @@ def get_local_driver(
import fasteners

edgedriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with edgedriver_fixing_lock:
if not _was_chromedriver_repaired(): # Works for Edge
if not _was_driver_repaired():
_repair_edgedriver(edge_version)
_mark_chromedriver_repaired() # Works for Edge
_mark_driver_repaired()
else:
if not _was_chromedriver_repaired(): # Works for Edge
if not _was_driver_repaired():
_repair_edgedriver(edge_version)
_mark_chromedriver_repaired() # Works for Edge
_mark_driver_repaired()
driver = Edge(
executable_path=LOCAL_EDGEDRIVER,
service_log_path=os.path.devnull,
Expand Down Expand Up @@ -1912,21 +1952,20 @@ def get_local_driver(
" executable: %s" % e
)
elif not is_chromedriver_on_path():
from seleniumbase.console_scripts import sb_install

args = " ".join(sys.argv)
if not ("-n" in sys.argv or " -n=" in args or args == "-c"):
# (Not multithreaded)
from seleniumbase.console_scripts import sb_install

sys_args = sys.argv # Save a copy of current sys args
print("\nWarning: chromedriver not found. Getting it now:")
sb_install.main(override="chromedriver")
sb_install.main(override="chromedriver latest")
sys.argv = sys_args # Put back the original sys args
else:
import fasteners
from seleniumbase.console_scripts import sb_install

chromedriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with chromedriver_fixing_lock:
if not is_chromedriver_on_path():
Expand All @@ -1935,7 +1974,7 @@ def get_local_driver(
"\nWarning: chromedriver not found. "
"Getting it now:"
)
sb_install.main(override="chromedriver")
sb_install.main(override="chromedriver latest")
sys.argv = sys_args # Put back original sys args
if not headless or "linux" not in PLATFORM:
try:
Expand Down Expand Up @@ -1976,6 +2015,16 @@ def get_local_driver(
raise Exception(e.msg) # Not an obvious fix. Raise.
else:
pass # Try upgrading ChromeDriver to match Chrome.
mcv = None # Major Chrome Version
if "Current browser version is " in e.msg:
line = e.msg.split("Current browser version is ")[1]
browser_version = line.split(" ")[0]
major_chrome_version = browser_version.split(".")[0]
if (
major_chrome_version.isnumeric()
and int(major_chrome_version) >= 86
):
mcv = major_chrome_version
headless = True
headless_options = _set_chrome_options(
browser_name,
Expand Down Expand Up @@ -2018,20 +2067,20 @@ def get_local_driver(
import fasteners

chromedriver_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with chromedriver_fixing_lock:
if not _was_chromedriver_repaired():
if not _was_driver_repaired():
_repair_chromedriver(
chrome_options, headless_options
chrome_options, headless_options, mcv
)
_mark_chromedriver_repaired()
_mark_driver_repaired()
else:
if not _was_chromedriver_repaired():
if not _was_driver_repaired():
_repair_chromedriver(
chrome_options, headless_options
chrome_options, headless_options, mcv
)
_mark_chromedriver_repaired()
_mark_driver_repaired()
if os.path.exists(LOCAL_CHROMEDRIVER):
if selenium4:
service = ChromeService(
Expand All @@ -2058,32 +2107,42 @@ def get_local_driver(
auto_upgrade_chromedriver = True
elif "Chrome version must be between" in e.msg:
auto_upgrade_chromedriver = True
mcv = None # Major Chrome Version
if "Current browser version is " in e.msg:
line = e.msg.split("Current browser version is ")[1]
browser_version = line.split(" ")[0]
major_chrome_version = browser_version.split(".")[0]
if (
major_chrome_version.isnumeric()
and int(major_chrome_version) >= 86
):
mcv = major_chrome_version
if auto_upgrade_chromedriver:
args = " ".join(sys.argv)
if "-n" in sys.argv or " -n=" in args or args == "-c":
import fasteners

chromedr_fixing_lock = fasteners.InterProcessLock(
constants.MultiBrowser.CHROMEDRIVER_FIXING_LOCK
constants.MultiBrowser.DRIVER_FIXING_LOCK
)
with chromedr_fixing_lock:
if not _was_chromedriver_repaired():
if not _was_driver_repaired():
try:
_repair_chromedriver(
chrome_options, chrome_options
chrome_options, chrome_options, mcv
)
_mark_chromedriver_repaired()
_mark_driver_repaired()
except Exception:
pass
else:
if not _was_chromedriver_repaired():
if not _was_driver_repaired():
try:
_repair_chromedriver(
chrome_options, chrome_options
chrome_options, chrome_options, mcv
)
except Exception:
pass
_mark_chromedriver_repaired()
_mark_driver_repaired()
try:
return webdriver.Chrome(options=chrome_options)
except Exception:
Expand Down

0 comments on commit d707826

Please sign in to comment.