CDP Mode: Patch 87

examples/cdp_mode/ReadMe.md

@@ -667,6 +667,8 @@ await tab.close()
 await tab.scroll_down(amount=25)
 await tab.scroll_up(amount=25)
 await tab.wait_for(selector="", text="", timeout=10)
+await tab.set_attributes(selector, attribute, value)
+await tab.internalize_links()
 await tab.download_file(url, filename=None)
 await tab.save_screenshot(
     filename="auto", format="png", full_page=False)

examples/cdp_mode/raw_priceline.py

@@ -1,9 +1,9 @@
 from seleniumbase import SB
 
-with SB(uc=True, test=True, locale="en", incognito=True) as sb:
+with SB(uc=True, test=True, locale="en") as sb:
     url = "https://www.priceline.com"
     sb.activate_cdp_mode(url)
-    sb.sleep(2.5)
+    sb.sleep(1.8)
     sb.click('input[name="endLocation"]')
     sb.sleep(1.2)
     location = "Portland, OR"
@@ -17,7 +17,12 @@
     sb.click('button[aria-label="Dismiss calendar"]')
     sb.sleep(0.5)
     sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
-    sb.sleep(5.5)
+    sb.sleep(0.5)
+    if sb.is_element_visible('[aria-label="Close Modal"]'):
+        sb.click('[aria-label="Close Modal"]')
+        sb.sleep(0.5)
+        sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
+    sb.sleep(4.8)
     if len(sb.cdp.get_tabs()) > 1:
         sb.cdp.close_active_tab()
         sb.cdp.switch_to_newest_tab()

examples/cdp_mode/raw_ralphlauren.py

@@ -0,0 +1,30 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale="en") as sb:
+    url = "https://www.ralphlauren.com.au/"
+    sb.activate_cdp_mode()
+    sb.open(url)
+    sb.sleep(1.2)
+    if not sb.is_element_present('[title="Locate Stores"]'):
+        sb.cdp.evaluate("window.location.reload();")
+        sb.sleep(1.2)
+    category = "women"
+    search = "Dresses"
+    sb.click('a[data-cgid="%s"]' % category)
+    sb.sleep(2.2)
+    sb.click('a:contains("%s")' % search)
+    sb.sleep(3.8)
+    for i in range(6):
+        sb.scroll_down(34)
+        sb.sleep(0.25)
+    print('*** Ralph Lauren Search for "%s":' % search)
+    unique_item_text = []
+    items = sb.find_elements('div.product-data')
+    for item in items:
+        description = item.querySelector("a.name-link")
+        if description and description.text not in unique_item_text:
+            unique_item_text.append(description.text)
+            print("* " + description.text)
+            price = item.querySelector('span[title="Price"]')
+            if price:
+                print("  (" + price.text.replace("   ", " ") + ")")

examples/cdp_mode/raw_totalwine.py

@@ -0,0 +1,31 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale="en") as sb:
+    url = "https://www.totalwine.com/"
+    sb.activate_cdp_mode()
+    sb.open(url)
+    sb.sleep(1.8)
+    search_box = 'input[data-at="header-search-text"]'
+    search = "The Land by Psagot Cabernet"
+    if not sb.is_element_present(search_box):
+        sb.cdp.evaluate("window.location.reload();")
+        sb.sleep(1.8)
+    sb.click_if_visible("#onetrust-close-btn-container button")
+    sb.sleep(0.5)
+    sb.click_if_visible('button[aria-label="Close modal"]')
+    sb.sleep(1.2)
+    sb.click(search_box)
+    sb.sleep(1.2)
+    sb.press_keys(search_box, search)
+    sb.sleep(0.6)
+    sb.click('button[data-at="header-search-button"]')
+    sb.sleep(1.8)
+    sb.click('img[data-at="product-search-productimage"]')
+    sb.sleep(2.2)
+    print('*** Total Wine Search for "%s":' % search)
+    print(sb.get_text('h1[data-at="product-name-title"]'))
+    print(sb.get_text('span[data-at="product-mix6price-text"]'))
+    print("Product Highlights:")
+    print(sb.get_text('p[class*="productInformationReview"]'))
+    print("Product Details:")
+    print(sb.get_text('div[data-at="origin-details-table-container"]'))

examples/cdp_mode/raw_zoro.py

@@ -0,0 +1,32 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale="en") as sb:
+    url = "https://www.zoro.com/"
+    sb.activate_cdp_mode()
+    sb.open(url)
+    sb.sleep(1.2)
+    search_box = "input#searchInput"
+    search = "Flir Thermal Camera"
+    required_text = "Camera"
+    if not sb.is_element_present(search_box):
+        sb.cdp.evaluate("window.location.reload();")
+        sb.sleep(1.2)
+    sb.click(search_box)
+    sb.sleep(1.2)
+    sb.press_keys(search_box, search)
+    sb.sleep(0.6)
+    sb.click('button[data-za="searchButton"]')
+    sb.sleep(3.8)
+    print('*** Zoro Search for "%s":' % search)
+    print('    (Results must contain "%s".)' % required_text)
+    unique_item_text = []
+    items = sb.find_elements('div[data-za="search-product-card"]')
+    for item in items:
+        if required_text in item.text:
+            description = item.querySelector('div[data-za="product-title"]')
+            if description and description.text not in unique_item_text:
+                unique_item_text.append(description.text)
+                print("* " + description.text)
+                price = item.querySelector("div.price-main")
+                if price:
+                    print("  (" + price.text + ")")

examples/presenter/uc_presentation_4.py

@@ -768,10 +768,10 @@ def test_presentation_4(self):
         )
         self.begin_presentation(filename="uc_presentation.html")
 
-        with SB(uc=True, test=True, locale="en", ad_block=True) as sb:
+        with SB(uc=True, test=True, locale="en") as sb:
             url = "https://www.priceline.com"
             sb.activate_cdp_mode(url)
-            sb.sleep(2.5)
+            sb.sleep(1.8)
             sb.click('input[name="endLocation"]')
             sb.sleep(1.2)
             location = "Portland, Oregon, US"
@@ -785,7 +785,12 @@ def test_presentation_4(self):
             sb.click('button[aria-label="Dismiss calendar"]')
             sb.sleep(0.5)
             sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
-            sb.sleep(5.5)
+            sb.sleep(0.5)
+            if sb.is_element_visible('[aria-label="Close Modal"]'):
+                sb.click('[aria-label="Close Modal"]')
+                sb.sleep(0.5)
+                sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]')
+            sb.sleep(4.8)
             if len(sb.cdp.get_tabs()) > 1:
                 sb.cdp.close_active_tab()
                 sb.cdp.switch_to_newest_tab()

help_docs/cdp_mode_methods.md

@@ -305,6 +305,8 @@ await tab.close()
 await tab.scroll_down(amount=25)
 await tab.scroll_up(amount=25)
 await tab.wait_for(selector="", text="", timeout=10)
+await tab.set_attributes(selector, attribute, value)
+await tab.internalize_links()
 await tab.download_file(url, filename=None)
 await tab.save_screenshot(
     filename="auto", format="png", full_page=False)

requirements.txt

@@ -76,7 +76,7 @@ rich>=14.2.0,<15
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage>=7.10.7;python_version<"3.10"
-coverage>=7.13.0;python_version>="3.10"
+coverage>=7.13.1;python_version>="3.10"
 pytest-cov>=7.0.0
 flake8==7.3.0
 mccabe==0.7.0

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.45.6"
+__version__ = "4.45.7"

seleniumbase/core/browser_launcher.py

@@ -2738,7 +2738,6 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-save-password-bubble")
     chrome_options.add_argument("--disable-single-click-autofill")
     chrome_options.add_argument("--allow-file-access-from-files")
-    chrome_options.add_argument("--disable-component-update")
     chrome_options.add_argument("--disable-prompt-on-repost")
     chrome_options.add_argument("--dns-prefetch-disable")
     chrome_options.add_argument("--disable-translate")
@@ -4793,7 +4792,6 @@ def get_local_driver(
         if devtools and not headless:
             edge_options.add_argument("--auto-open-devtools-for-tabs")
         edge_options.add_argument("--allow-file-access-from-files")
-        edge_options.add_argument("--disable-component-update")
         edge_options.add_argument("--allow-insecure-localhost")
         edge_options.add_argument("--allow-running-insecure-content")
         if user_agent:

seleniumbase/core/sb_cdp.py

@@ -1664,13 +1664,15 @@ def set_attributes(self, selector, attribute, value):
         css_selector = self.__convert_to_css_if_xpath(selector)
         css_selector = re.escape(css_selector)  # Add "\\" to special chars
         css_selector = js_utils.escape_quotes_if_needed(css_selector)
-        js_code = """var $elements = document.querySelectorAll('%s');
-                  var index = 0, length = $elements.length;
-                  for(; index < length; index++){
-                  $elements[index].setAttribute('%s','%s');}""" % (
-            css_selector,
-            attribute,
-            value,
+        js_code = (
+            """var $elements = document.querySelectorAll('%s');
+            var index = 0, length = $elements.length;
+            for(; index < length; index++){
+            $elements[index].setAttribute('%s','%s');}""" % (
+                css_selector,
+                attribute,
+                value,
+            )
         )
         with suppress(Exception):
             self.loop.run_until_complete(self.page.evaluate(js_code))
@@ -1965,8 +1967,8 @@ def _on_a_cf_turnstile_page(self, source=None):
             return True
         return False
 
-    def _on_a_g_recaptcha_page(self, source=None):
-        time.sleep(0.4)
+    def _on_a_g_recaptcha_page(self, *args, **kwargs):
+        time.sleep(0.4)  # reCAPTCHA may need a moment to appear
         self.loop.run_until_complete(self.page.wait())
         source = self.get_page_source()
         if (

seleniumbase/undetected/cdp_driver/browser.py

@@ -580,7 +580,7 @@ async def start(self=None) -> Browser:
             )  # noqa
         exe = self.config.browser_executable_path
         params = self.config()
-        logger.info(
+        logger.debug(
             "Starting\n\texecutable :%s\n\narguments:\n%s",
             exe,
             "\n\t".join(params),
@@ -636,7 +636,7 @@ async def start(self=None) -> Browser:
             self.info.webSocketDebuggerUrl, browser=self
         )
         if self.config.autodiscover_targets:
-            logger.info("Enabling autodiscover targets")
+            logger.debug("Enabling autodiscover targets")
             self.connection.handlers[cdp.target.TargetInfoChanged] = [
                 self._handle_target_update
             ]
@@ -863,15 +863,15 @@ def stop(self):
         for _ in range(3):
             try:
                 self._process.terminate()
-                logger.info(
+                logger.debug(
                     "Terminated browser with pid %d successfully."
                     % self._process.pid
                 )
                 break
             except (Exception,):
                 try:
                     self._process.kill()
-                    logger.info(
+                    logger.debug(
                         "Killed browser with pid %d successfully."
                         % self._process.pid
                     )
@@ -880,14 +880,14 @@ def stop(self):
                     try:
                         if hasattr(self, "browser_process_pid"):
                             os.kill(self._process_pid, 15)
-                            logger.info(
+                            logger.debug(
                                 "Killed browser with pid %d "
                                 "using signal 15 successfully."
                                 % self._process.pid
                             )
                             break
                     except (TypeError,):
-                        logger.info("typerror", exc_info=True)
+                        logger.info("TypeError", exc_info=True)
                         pass
                     except (PermissionError,):
                         logger.info(
@@ -896,7 +896,7 @@ def stop(self):
                         )
                         pass
                     except (ProcessLookupError,):
-                        logger.info("Process lookup failure!")
+                        logger.info("ProcessLookupError")
                         pass
                     except (Exception,):
                         raise

seleniumbase/undetected/cdp_driver/config.py

@@ -207,7 +207,6 @@ def __init__(
             "--disable-top-sites",
             "--disable-translate",
             "--dns-prefetch-disable",
-            "--disable-component-update",
             "--disable-renderer-backgrounding",
             "--disable-dev-shm-usage",
         ]

seleniumbase/undetected/cdp_driver/element.py

@@ -372,7 +372,10 @@ async def click_async(self):
         arguments = [cdp.runtime.CallArgument(
             object_id=self._remote_object.object_id
         )]
-        await self.flash_async(0.25)
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            await self.flash_async(0.25)
         await self._tab.send(
             cdp.runtime.call_function_on(
                 "(el) => el.click()",
@@ -501,7 +504,10 @@ async def mouse_click_async(
             logger.warning("Could not calculate box model for %s", self)
             return
         logger.debug("Clicking on location: %.2f, %.2f" % center)
-        asyncio.create_task(self.flash_async(0.25))
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            asyncio.create_task(self.flash_async(0.25))
         asyncio.create_task(
             self._tab.send(
                 cdp.input_.dispatch_mouse_event(
@@ -560,12 +566,15 @@ async def mouse_click_with_offset_async(
             logger.debug("Clicking on location: %.2f, %.2f" % center_pos)
         else:
             logger.debug("Clicking on location: %.2f, %.2f" % (x_pos, y_pos))
-        asyncio.create_task(
-            self.flash_async(
-                x_offset=x_offset - (width / 2),
-                y_offset=y_offset - (height / 2),
-            ),
-        )
+        script = 'sessionStorage.getItem("pxsid") !== null;'
+        using_px = await self.tab.evaluate(script)
+        if not using_px:
+            asyncio.create_task(
+                self.flash_async(
+                    x_offset=x_offset - (width / 2),
+                    y_offset=y_offset - (height / 2),
+                ),
+            )
         asyncio.create_task(
             self._tab.send(
                 cdp.input_.dispatch_mouse_event(

seleniumbase/undetected/cdp_driver/tab.py

@@ -1092,6 +1092,39 @@ async def wait_for(
                 await self.sleep(0.068)
             return item
 
+    async def set_attributes(self, selector, attribute, value):
+        """This method uses JavaScript to set/update a common attribute.
+        All matching selectors from querySelectorAll() are used.
+        Example => (Make all links on a website redirect to Google):
+        self.set_attributes("a", "href", "https://google.com")"""
+        attribute = re.escape(attribute)
+        attribute = js_utils.escape_quotes_if_needed(attribute)
+        value = re.escape(value)
+        value = js_utils.escape_quotes_if_needed(value)
+        if selector.startswith(("/", "./", "(")):
+            with suppress(Exception):
+                selector = js_utils.convert_to_css_selector(selector, "xpath")
+        css_selector = selector
+        css_selector = re.escape(css_selector)  # Add "\\" to special chars
+        css_selector = js_utils.escape_quotes_if_needed(css_selector)
+        js_code = (
+            """var $elements = document.querySelectorAll('%s');
+            var index = 0, length = $elements.length;
+            for(; index < length; index++){
+            $elements[index].setAttribute('%s','%s');}""" % (
+                css_selector,
+                attribute,
+                value,
+            )
+        )
+        with suppress(Exception):
+            await self.evaluate(js_code)
+
+    async def internalize_links(self):
+        """All `target="_blank"` links become `target="_self"`.
+        This prevents those links from opening in a new tab."""
+        await self.set_attributes('[target="_blank"]', "target", "_self")
+
     async def download_file(
         self, url: str, filename: Optional[PathLike] = None
     ):
@@ -1336,8 +1369,8 @@ async def __on_a_cf_turnstile_page(self, source=None):
             return True
         return False
 
-    async def __on_a_g_recaptcha_page(self, source=None):
-        await self.sleep(0.4)
+    async def __on_a_g_recaptcha_page(self, *args, **kwargs):
+        await self.sleep(0.4)  # reCAPTCHA may need a moment to appear
         source = await self.get_html()
         if (
             (