CDP Mode: Patch 73

examples/cdp_mode/raw_ad_blocking.py

@@ -7,15 +7,15 @@ async def block_urls(tab):
     await tab.send(mycdp.network.enable())
     await tab.send(mycdp.network.set_blocked_urls(
         urls=[
-            "*googlesyndication.com*",
-            "*googletagmanager.com*",
-            "*google-analytics.com*",
-            "*amazon-adsystem.com*",
-            "*adsafeprotected.com*",
-            "*doubleclick.net*",
-            "*fastclick.net*",
-            "*snigelweb.com*",
-            "*2mdn.net*",
+            "*.googlesyndication.com*",
+            "*.googletagmanager.com*",
+            "*.google-analytics.com*",
+            "*.amazon-adsystem.com*",
+            "*.adsafeprotected.com*",
+            "*.doubleclick.net*",
+            "*.fastclick.net*",
+            "*.snigelweb.com*",
+            "*.2mdn.net*",
         ]
     ))
 

examples/cdp_mode/raw_cdp_gitlab.py

@@ -2,7 +2,6 @@
 
 url = "https://gitlab.com/users/sign_in"
 sb = sb_cdp.Chrome(url, incognito=True)
-sb.sleep(2.2)
 sb.solve_captcha()
 sb.highlight('h1:contains("GitLab")')
 sb.highlight('button:contains("Sign in")')

examples/cdp_mode/raw_gitlab.py

@@ -3,7 +3,6 @@
 with SB(uc=True, test=True, locale="en") as sb:
     url = "https://gitlab.com/users/sign_in"
     sb.activate_cdp_mode(url)
-    sb.sleep(2)
     sb.solve_captcha()
     # (The rest is for testing and demo purposes)
     sb.assert_text("Username", '[for="user_login"]', timeout=3)

examples/cdp_mode/raw_glassdoor.py

@@ -3,7 +3,7 @@
 with SB(uc=True, test=True, ad_block=True) as sb:
     url = "https://www.glassdoor.com/Reviews/index.htm"
     sb.activate_cdp_mode(url)
-    sb.sleep(2.2)
+    sb.sleep(1.2)
     sb.solve_captcha()
     sb.highlight('[data-test="global-nav-glassdoor-logo"]')
     sb.highlight('[data-test="site-header-companies"]')

examples/cdp_mode/raw_planetmc.py

@@ -3,7 +3,7 @@
 with SB(uc=True, test=True) as sb:
     url = "www.planetminecraft.com/account/sign_in/"
     sb.activate_cdp_mode(url)
-    sb.sleep(2)
+    sb.sleep(1.2)
     sb.solve_captcha()
     sb.wait_for_element_absent("input[disabled]")
     sb.sleep(2)

examples/cdp_mode/raw_science.py

@@ -4,12 +4,12 @@
     url = "https://earth.esa.int/eogateway/search"
     sb.activate_cdp_mode(url)
     sb.sleep(1)
-    sb.cdp.click_if_visible('button:contains("Accept cookies")')
+    sb.click_if_visible('button:contains("Accept cookies")')
     for i in range(20):
-        sb.cdp.scroll_to_bottom()
-        sb.cdp.click_if_visible('button:contains("READ MORE")')
+        sb.scroll_to_bottom()
+        sb.click_if_visible('button:contains("READ MORE")')
     sb.sleep(1)
-    elements = sb.cdp.find_elements("h4 a span")
+    elements = sb.find_elements("h4 a span")
     for element in elements:
         print(element.text)
     print("*** Total entries: %s" % len(elements))

examples/cdp_mode/raw_timezone.py

@@ -26,7 +26,6 @@ def main():
     )
     sb.add_handler(mycdp.fetch.RequestPaused, request_paused_handler)
     sb.remove_elements("#right-sidebar")
-    sb.remove_elements('[id*="Footer"]')
     sb.sleep(6)
     sb.driver.stop()
 

examples/cdp_mode/raw_timezone_sb.py

@@ -5,9 +5,7 @@
     url = "https://www.randymajors.org/what-time-zone-am-i-in"
     sb.activate_cdp_mode(url, tzone="Asia/Kolkata", geoloc=(26.863, 80.94))
     sb.remove_elements("#right-sidebar")
-    sb.remove_elements('[id*="Footer"]')
     sb.sleep(5)
     sb.cdp.open(url, tzone="Asia/Tokyo", geoloc=(35.050681, 136.844728))
     sb.remove_elements("#right-sidebar")
-    sb.remove_elements('[id*="Footer"]')
     sb.sleep(5)

requirements.txt

@@ -90,7 +90,7 @@ rich>=14.2.0,<15
 
 coverage>=7.6.1;python_version<"3.9"
 coverage>=7.10.7;python_version>="3.9" and python_version<"3.10"
-coverage>=7.11.1;python_version>="3.10"
+coverage>=7.11.3;python_version>="3.10"
 pytest-cov>=5.0.0;python_version<"3.9"
 pytest-cov>=7.0.0;python_version>="3.9"
 flake8==5.0.4;python_version<"3.9"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.44.8"
+__version__ = "4.44.9"

seleniumbase/core/browser_launcher.py

@@ -938,6 +938,14 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     driver.solve_captcha = CDPM.solve_captcha
     driver.find_element_by_text = CDPM.find_element_by_text
     driver._is_using_cdp = True
+    if (
+        hasattr(sb_config, "_cdp_proxy")
+        and sb_config._cdp_proxy
+        and "@" in sb_config._cdp_proxy
+    ):
+        time.sleep(0.077)
+        loop.run_until_complete(page.wait(0.25))
+        time.sleep(0.022)
 
 
 def uc_activate_cdp_mode(driver, url=None, **kwargs):
@@ -2746,7 +2754,7 @@ def _set_chrome_options(
     d_f_string = ",".join(included_disabled_features)
     chrome_options.add_argument("--disable-features=%s" % d_f_string)
     chrome_options.add_argument("--enable-unsafe-extension-debugging")
-    if proxy_auth:
+    if proxy_string:
         chrome_options.add_argument("--test-type")
     if proxy_auth or sb_config._ext_dirs:
         if not is_using_uc(undetectable, browser_name):
@@ -2770,7 +2778,7 @@ def _set_chrome_options(
         chrome_options.add_argument("--disable-popup-blocking")
         # Skip remaining options that trigger anti-bot services
         return chrome_options
-    if not proxy_auth:
+    if not proxy_string:
         chrome_options.add_argument("--test-type")
     chrome_options.add_argument("--log-level=3")
     chrome_options.add_argument("--no-first-run")
@@ -5812,6 +5820,7 @@ def get_local_driver(
                             driver, *args, **kwargs
                         )
                     )
+                    driver.activate_cdp_mode = driver.uc_activate_cdp_mode
                     driver.uc_open_with_cdp_mode = (
                         lambda *args, **kwargs: uc_open_with_cdp_mode(
                             driver, *args, **kwargs

seleniumbase/core/sb_cdp.py

@@ -1823,7 +1823,11 @@ def _on_a_cf_turnstile_page(self, source=None):
             time.sleep(0.2)
             source = self.get_page_source()
         if (
-            'data-callback="onCaptchaSuccess"' in source
+            (
+                'data-callback="onCaptchaSuccess"' in source
+                and 'title="reCAPTCHA"' not in source
+                and 'id="recaptcha-token"' not in source
+            )
             or "/challenge-platform/scripts/" in source
             or 'id="challenge-widget-' in source
             or "challenges.cloudf" in source
@@ -1833,16 +1837,21 @@ def _on_a_cf_turnstile_page(self, source=None):
         return False
 
     def _on_a_g_recaptcha_page(self, source=None):
-        if not source or len(source) < 400:
-            time.sleep(0.2)
-            source = self.get_page_source()
+        time.sleep(0.25)
+        self.loop.run_until_complete(self.page.wait(0.25))
+        source = self.get_page_source()
         if (
-            'id="recaptcha-token"' in source
-            or 'title="reCAPTCHA"' in source
+            (
+                'id="recaptcha-token"' in source
+                or 'title="reCAPTCHA"' in source
+            )
+            and self.is_element_visible('iframe[title="reCAPTCHA"]')
         ):
+            self.loop.run_until_complete(self.page.wait(0.1))
             return True
         elif "/recaptcha/api.js" in source:
             time.sleep(1.6)  # Still loading
+            self.loop.run_until_complete(self.page.wait(0.1))
             return True
         return False
 
@@ -1852,8 +1861,10 @@ def __gui_click_recaptcha(self, use_cdp=False):
             selector = 'iframe[title="reCAPTCHA"]'
         else:
             return
+        time.sleep(0.25)
+        self.loop.run_until_complete(self.page.wait())
+        time.sleep(0.25)
         with suppress(Exception):
-            time.sleep(0.08)
             element_rect = self.get_gui_element_rect(selector, timeout=1)
             e_x = element_rect["x"]
             e_y = element_rect["y"]
@@ -1884,7 +1895,9 @@ def gui_click_captcha(self):
 
     def __click_captcha(self, use_cdp=False):
         """Uses PyAutoGUI unless use_cdp == True"""
-        self.sleep(0.056)
+        self.sleep(0.075)
+        self.loop.run_until_complete(self.page.wait())
+        self.sleep(0.025)
         source = self.get_page_source()
         if self._on_a_cf_turnstile_page(source):
             pass
@@ -1894,61 +1907,32 @@ def __click_captcha(self, use_cdp=False):
         else:
             return
         selector = None
-        if (
-            self.is_element_present('[name*="cf-turnstile-"]')
-            and self.is_element_present("#challenge-form div > div")
-        ):
+        if self.is_element_present('[class="cf-turnstile"]'):
+            selector = '[class="cf-turnstile"]'
+        elif self.is_element_present("#challenge-form div > div"):
             selector = "#challenge-form div > div"
-        elif (
-            self.is_element_present('[name*="cf-turnstile-"]')
-            and self.is_element_present(
-                '[style="display: grid;"] div div'
-            )
-        ):
+        elif self.is_element_present('[style="display: grid;"] div div'):
             selector = '[style="display: grid;"] div div'
-        elif (
-            self.is_element_present('[name*="cf-turnstile-"]')
-            and self.is_element_present("[class*=spacer] + div div")
-        ):
+        elif self.is_element_present("[class*=spacer] + div div"):
             selector = '[class*=spacer] + div div'
-        elif (
-            self.is_element_present('[name*="cf-turnstile-"]')
-            and self.is_element_present(".spacer div:not([class])")
-        ):
+        elif self.is_element_present(".spacer div:not([class])"):
             selector = ".spacer div:not([class])"
-        elif (
-            self.is_element_present('script[src*="challenges.c"]')
-            and self.is_element_present(
-                '[data-testid*="challenge-"] div'
-            )
-        ):
+        elif self.is_element_present('[data-testid*="challenge-"] div'):
             selector = '[data-testid*="challenge-"] div'
-        elif self.is_element_present(
-            "div#turnstile-widget div:not([class])"
-        ):
+        elif self.is_element_present("div#turnstile-widget div:not([class])"):
             selector = "div#turnstile-widget div:not([class])"
         elif self.is_element_present("ngx-turnstile div:not([class])"):
             selector = "ngx-turnstile div:not([class])"
         elif self.is_element_present(
             'form div:not([class]):has(input[name*="cf-turn"])'
         ):
             selector = 'form div:not([class]):has(input[name*="cf-turn"])'
-        elif (
-            self.is_element_present('[src*="/turnstile/"]')
-            and self.is_element_present("form div:not(:has(*))")
-        ):
+        elif self.is_element_present("form div:not(:has(*))"):
             selector = "form div:not(:has(*))"
-        elif (
-            self.is_element_present('[src*="/turnstile/"]')
-            and self.is_element_present(
-                "body > div#check > div:not([class])"
-            )
-        ):
+        elif self.is_element_present("body > div#check > div:not([class])"):
             selector = "body > div#check > div:not([class])"
         elif self.is_element_present(".cf-turnstile-wrapper"):
             selector = ".cf-turnstile-wrapper"
-        elif self.is_element_present('[class="cf-turnstile"]'):
-            selector = '[class="cf-turnstile"]'
         elif self.is_element_present(
             '[id*="turnstile"] div:not([class])'
         ):
@@ -2048,7 +2032,7 @@ def __click_captcha(self, use_cdp=False):
                 self.loop.run_until_complete(self.page.evaluate(script))
                 self.loop.run_until_complete(self.page.wait())
         with suppress(Exception):
-            time.sleep(0.08)
+            time.sleep(0.05)
             element_rect = self.get_gui_element_rect(selector, timeout=1)
             e_x = element_rect["x"]
             e_y = element_rect["y"]
@@ -2059,15 +2043,17 @@ def __click_captcha(self, use_cdp=False):
             x = e_x + x_offset
             y = e_y + y_offset
             sb_config._saved_cf_x_y = (x, y)
-            time.sleep(0.08)
+            time.sleep(0.05)
+            if hasattr(sb_config, "_cdp_proxy") and sb_config._cdp_proxy:
+                time.sleep(0.22)  # CAPTCHA may load slower with proxy
             if use_cdp:
                 self.sleep(0.03)
                 gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
                 with gui_lock:  # Prevent issues with multiple processes
                     self.bring_active_window_to_front()
-                    time.sleep(0.056)
+                    time.sleep(0.05)
                     self.click_with_offset(selector, x_offset, y_offset)
-                    time.sleep(0.056)
+                    time.sleep(0.05)
             else:
                 self.gui_click_x_y(x, y)
 

seleniumbase/fixtures/base_case.py

@@ -227,6 +227,28 @@ def open(self, url):
         if self.__is_cdp_swap_needed():
             self.cdp.open(url)
             return
+        elif (
+            hasattr(self.driver, "_is_using_uc")
+            and self.driver._is_using_uc
+            and "@" in self.proxy_string
+            and (
+                not hasattr(self.driver, "_is_using_cdp")
+                or not self.driver._is_using_cdp
+            )
+        ):
+            # Auth in UC Mode requires CDP Mode
+            logging.info("UC Mode requires CDP Mode for auth. Activating now.")
+            self.activate_cdp_mode(url)
+            return
+        elif (
+            hasattr(self.driver, "_is_using_uc")
+            and self.driver._is_using_uc
+            and hasattr(self.driver, "_is_using_cdp")
+            and self.driver._is_using_cdp
+        ):
+            self.disconnect()
+            self.cdp.open(url)
+            return
         self._check_browser()
         if self.__needs_minimum_wait():
             time.sleep(0.04)
@@ -5005,7 +5027,7 @@ def activate_cdp_mode(self, url=None, **kwargs):
                 self.driver.connect()
             current_url = self.get_current_url()
             if not current_url.startswith(("about", "data", "chrome")):
-                self.open("about:blank")
+                self.driver.get("about:blank")
             self.driver.uc_open_with_cdp_mode(url, **kwargs)
         else:
             self.get_new_driver(undetectable=True)
@@ -5015,6 +5037,13 @@ def activate_cdp_mode(self, url=None, **kwargs):
             self.solve_captcha = self.cdp.solve_captcha
         if hasattr(self.cdp, "find_element_by_text"):
             self.find_element_by_text = self.cdp.find_element_by_text
+        if (
+            hasattr(sb_config, "_cdp_proxy")
+            and sb_config._cdp_proxy
+            and "@" in sb_config._cdp_proxy
+        ):
+            with suppress(Exception):
+                self.cdp.loop.run_until_complete(self.cdp.page.wait(0.25))
         self.undetectable = True
 
     def activate_recorder(self):
@@ -14527,7 +14556,8 @@ def __activate_virtual_display(self):
                 sb_config._virtual_display = self._xvfb_display
                 if "DISPLAY" not in os.environ.keys():
                     print(
-                        "\nX11 display failed! Will use regular xvfb!"
+                        "\n  X11 display failed! Is Xvfb installed? "
+                        "\n  Try this: `sudo apt install -y xvfb`"
                     )
                     self.__activate_standard_virtual_display()
                 else:
@@ -14540,7 +14570,10 @@ def __activate_virtual_display(self):
                     print("\n" + str(e.msg))
                 else:
                     print(e)
-                print("\nX11 display failed! Will use regular xvfb!")
+                print(
+                    "\n  X11 display failed! Is Xvfb installed? "
+                    "\n  Try this: `sudo apt install -y xvfb`"
+                )
                 self.__activate_standard_virtual_display()
                 return
             pyautogui_is_installed = False