diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 8b13bf59fd6..a60cae1896f 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -667,6 +667,8 @@ await tab.close() await tab.scroll_down(amount=25) await tab.scroll_up(amount=25) await tab.wait_for(selector="", text="", timeout=10) +await tab.set_attributes(selector, attribute, value) +await tab.internalize_links() await tab.download_file(url, filename=None) await tab.save_screenshot( filename="auto", format="png", full_page=False) diff --git a/examples/cdp_mode/raw_priceline.py b/examples/cdp_mode/raw_priceline.py index 3703bf4a8c7..d9757fa778d 100644 --- a/examples/cdp_mode/raw_priceline.py +++ b/examples/cdp_mode/raw_priceline.py @@ -1,9 +1,9 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale="en", incognito=True) as sb: +with SB(uc=True, test=True, locale="en") as sb: url = "https://www.priceline.com" sb.activate_cdp_mode(url) - sb.sleep(2.5) + sb.sleep(1.8) sb.click('input[name="endLocation"]') sb.sleep(1.2) location = "Portland, OR" @@ -17,7 +17,12 @@ sb.click('button[aria-label="Dismiss calendar"]') sb.sleep(0.5) sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') - sb.sleep(5.5) + sb.sleep(0.5) + if sb.is_element_visible('[aria-label="Close Modal"]'): + sb.click('[aria-label="Close Modal"]') + sb.sleep(0.5) + sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') + sb.sleep(4.8) if len(sb.cdp.get_tabs()) > 1: sb.cdp.close_active_tab() sb.cdp.switch_to_newest_tab() diff --git a/examples/cdp_mode/raw_ralphlauren.py b/examples/cdp_mode/raw_ralphlauren.py new file mode 100644 index 00000000000..99da77ad065 --- /dev/null +++ b/examples/cdp_mode/raw_ralphlauren.py @@ -0,0 +1,30 @@ +from seleniumbase import SB + +with SB(uc=True, test=True, locale="en") as sb: + url = "https://www.ralphlauren.com.au/" + sb.activate_cdp_mode() + sb.open(url) + sb.sleep(1.2) + if not sb.is_element_present('[title="Locate Stores"]'): + sb.cdp.evaluate("window.location.reload();") + sb.sleep(1.2) + category = "women" + search = "Dresses" + sb.click('a[data-cgid="%s"]' % category) + sb.sleep(2.2) + sb.click('a:contains("%s")' % search) + sb.sleep(3.8) + for i in range(6): + sb.scroll_down(34) + sb.sleep(0.25) + print('*** Ralph Lauren Search for "%s":' % search) + unique_item_text = [] + items = sb.find_elements('div.product-data') + for item in items: + description = item.querySelector("a.name-link") + if description and description.text not in unique_item_text: + unique_item_text.append(description.text) + print("* " + description.text) + price = item.querySelector('span[title="Price"]') + if price: + print(" (" + price.text.replace(" ", " ") + ")") diff --git a/examples/cdp_mode/raw_totalwine.py b/examples/cdp_mode/raw_totalwine.py new file mode 100644 index 00000000000..1292be3ad52 --- /dev/null +++ b/examples/cdp_mode/raw_totalwine.py @@ -0,0 +1,31 @@ +from seleniumbase import SB + +with SB(uc=True, test=True, locale="en") as sb: + url = "https://www.totalwine.com/" + sb.activate_cdp_mode() + sb.open(url) + sb.sleep(1.8) + search_box = 'input[data-at="header-search-text"]' + search = "The Land by Psagot Cabernet" + if not sb.is_element_present(search_box): + sb.cdp.evaluate("window.location.reload();") + sb.sleep(1.8) + sb.click_if_visible("#onetrust-close-btn-container button") + sb.sleep(0.5) + sb.click_if_visible('button[aria-label="Close modal"]') + sb.sleep(1.2) + sb.click(search_box) + sb.sleep(1.2) + sb.press_keys(search_box, search) + sb.sleep(0.6) + sb.click('button[data-at="header-search-button"]') + sb.sleep(1.8) + sb.click('img[data-at="product-search-productimage"]') + sb.sleep(2.2) + print('*** Total Wine Search for "%s":' % search) + print(sb.get_text('h1[data-at="product-name-title"]')) + print(sb.get_text('span[data-at="product-mix6price-text"]')) + print("Product Highlights:") + print(sb.get_text('p[class*="productInformationReview"]')) + print("Product Details:") + print(sb.get_text('div[data-at="origin-details-table-container"]')) diff --git a/examples/cdp_mode/raw_zoro.py b/examples/cdp_mode/raw_zoro.py new file mode 100644 index 00000000000..df673467816 --- /dev/null +++ b/examples/cdp_mode/raw_zoro.py @@ -0,0 +1,32 @@ +from seleniumbase import SB + +with SB(uc=True, test=True, locale="en") as sb: + url = "https://www.zoro.com/" + sb.activate_cdp_mode() + sb.open(url) + sb.sleep(1.2) + search_box = "input#searchInput" + search = "Flir Thermal Camera" + required_text = "Camera" + if not sb.is_element_present(search_box): + sb.cdp.evaluate("window.location.reload();") + sb.sleep(1.2) + sb.click(search_box) + sb.sleep(1.2) + sb.press_keys(search_box, search) + sb.sleep(0.6) + sb.click('button[data-za="searchButton"]') + sb.sleep(3.8) + print('*** Zoro Search for "%s":' % search) + print(' (Results must contain "%s".)' % required_text) + unique_item_text = [] + items = sb.find_elements('div[data-za="search-product-card"]') + for item in items: + if required_text in item.text: + description = item.querySelector('div[data-za="product-title"]') + if description and description.text not in unique_item_text: + unique_item_text.append(description.text) + print("* " + description.text) + price = item.querySelector("div.price-main") + if price: + print(" (" + price.text + ")") diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index 59b26be4e5b..bb093c2ae3f 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -768,10 +768,10 @@ def test_presentation_4(self): ) self.begin_presentation(filename="uc_presentation.html") - with SB(uc=True, test=True, locale="en", ad_block=True) as sb: + with SB(uc=True, test=True, locale="en") as sb: url = "https://www.priceline.com" sb.activate_cdp_mode(url) - sb.sleep(2.5) + sb.sleep(1.8) sb.click('input[name="endLocation"]') sb.sleep(1.2) location = "Portland, Oregon, US" @@ -785,7 +785,12 @@ def test_presentation_4(self): sb.click('button[aria-label="Dismiss calendar"]') sb.sleep(0.5) sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') - sb.sleep(5.5) + sb.sleep(0.5) + if sb.is_element_visible('[aria-label="Close Modal"]'): + sb.click('[aria-label="Close Modal"]') + sb.sleep(0.5) + sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') + sb.sleep(4.8) if len(sb.cdp.get_tabs()) > 1: sb.cdp.close_active_tab() sb.cdp.switch_to_newest_tab() diff --git a/help_docs/cdp_mode_methods.md b/help_docs/cdp_mode_methods.md index e35d12d8b75..1092a1b8a2c 100644 --- a/help_docs/cdp_mode_methods.md +++ b/help_docs/cdp_mode_methods.md @@ -305,6 +305,8 @@ await tab.close() await tab.scroll_down(amount=25) await tab.scroll_up(amount=25) await tab.wait_for(selector="", text="", timeout=10) +await tab.set_attributes(selector, attribute, value) +await tab.internalize_links() await tab.download_file(url, filename=None) await tab.save_screenshot( filename="auto", format="png", full_page=False) diff --git a/requirements.txt b/requirements.txt index 53cc6c7fc1b..9e97077fa3f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -76,7 +76,7 @@ rich>=14.2.0,<15 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.10.7;python_version<"3.10" -coverage>=7.13.0;python_version>="3.10" +coverage>=7.13.1;python_version>="3.10" pytest-cov>=7.0.0 flake8==7.3.0 mccabe==0.7.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 542e47542cb..0513ea6fd70 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.45.6" +__version__ = "4.45.7" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index ad7b0a9763f..c604515d416 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -2738,7 +2738,6 @@ def _set_chrome_options( chrome_options.add_argument("--disable-save-password-bubble") chrome_options.add_argument("--disable-single-click-autofill") chrome_options.add_argument("--allow-file-access-from-files") - chrome_options.add_argument("--disable-component-update") chrome_options.add_argument("--disable-prompt-on-repost") chrome_options.add_argument("--dns-prefetch-disable") chrome_options.add_argument("--disable-translate") @@ -4793,7 +4792,6 @@ def get_local_driver( if devtools and not headless: edge_options.add_argument("--auto-open-devtools-for-tabs") edge_options.add_argument("--allow-file-access-from-files") - edge_options.add_argument("--disable-component-update") edge_options.add_argument("--allow-insecure-localhost") edge_options.add_argument("--allow-running-insecure-content") if user_agent: diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index c9f1d91f999..49462823a0c 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -1664,13 +1664,15 @@ def set_attributes(self, selector, attribute, value): css_selector = self.__convert_to_css_if_xpath(selector) css_selector = re.escape(css_selector) # Add "\\" to special chars css_selector = js_utils.escape_quotes_if_needed(css_selector) - js_code = """var $elements = document.querySelectorAll('%s'); - var index = 0, length = $elements.length; - for(; index < length; index++){ - $elements[index].setAttribute('%s','%s');}""" % ( - css_selector, - attribute, - value, + js_code = ( + """var $elements = document.querySelectorAll('%s'); + var index = 0, length = $elements.length; + for(; index < length; index++){ + $elements[index].setAttribute('%s','%s');}""" % ( + css_selector, + attribute, + value, + ) ) with suppress(Exception): self.loop.run_until_complete(self.page.evaluate(js_code)) @@ -1965,8 +1967,8 @@ def _on_a_cf_turnstile_page(self, source=None): return True return False - def _on_a_g_recaptcha_page(self, source=None): - time.sleep(0.4) + def _on_a_g_recaptcha_page(self, *args, **kwargs): + time.sleep(0.4) # reCAPTCHA may need a moment to appear self.loop.run_until_complete(self.page.wait()) source = self.get_page_source() if ( diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index a375eb34734..1f380979b0a 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -580,7 +580,7 @@ async def start(self=None) -> Browser: ) # noqa exe = self.config.browser_executable_path params = self.config() - logger.info( + logger.debug( "Starting\n\texecutable :%s\n\narguments:\n%s", exe, "\n\t".join(params), @@ -636,7 +636,7 @@ async def start(self=None) -> Browser: self.info.webSocketDebuggerUrl, browser=self ) if self.config.autodiscover_targets: - logger.info("Enabling autodiscover targets") + logger.debug("Enabling autodiscover targets") self.connection.handlers[cdp.target.TargetInfoChanged] = [ self._handle_target_update ] @@ -863,7 +863,7 @@ def stop(self): for _ in range(3): try: self._process.terminate() - logger.info( + logger.debug( "Terminated browser with pid %d successfully." % self._process.pid ) @@ -871,7 +871,7 @@ def stop(self): except (Exception,): try: self._process.kill() - logger.info( + logger.debug( "Killed browser with pid %d successfully." % self._process.pid ) @@ -880,14 +880,14 @@ def stop(self): try: if hasattr(self, "browser_process_pid"): os.kill(self._process_pid, 15) - logger.info( + logger.debug( "Killed browser with pid %d " "using signal 15 successfully." % self._process.pid ) break except (TypeError,): - logger.info("typerror", exc_info=True) + logger.info("TypeError", exc_info=True) pass except (PermissionError,): logger.info( @@ -896,7 +896,7 @@ def stop(self): ) pass except (ProcessLookupError,): - logger.info("Process lookup failure!") + logger.info("ProcessLookupError") pass except (Exception,): raise diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index 06cedd944f5..b5279ac32b5 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -207,7 +207,6 @@ def __init__( "--disable-top-sites", "--disable-translate", "--dns-prefetch-disable", - "--disable-component-update", "--disable-renderer-backgrounding", "--disable-dev-shm-usage", ] diff --git a/seleniumbase/undetected/cdp_driver/element.py b/seleniumbase/undetected/cdp_driver/element.py index 023c651bf0a..93f78bfee5e 100644 --- a/seleniumbase/undetected/cdp_driver/element.py +++ b/seleniumbase/undetected/cdp_driver/element.py @@ -372,7 +372,10 @@ async def click_async(self): arguments = [cdp.runtime.CallArgument( object_id=self._remote_object.object_id )] - await self.flash_async(0.25) + script = 'sessionStorage.getItem("pxsid") !== null;' + using_px = await self.tab.evaluate(script) + if not using_px: + await self.flash_async(0.25) await self._tab.send( cdp.runtime.call_function_on( "(el) => el.click()", @@ -501,7 +504,10 @@ async def mouse_click_async( logger.warning("Could not calculate box model for %s", self) return logger.debug("Clicking on location: %.2f, %.2f" % center) - asyncio.create_task(self.flash_async(0.25)) + script = 'sessionStorage.getItem("pxsid") !== null;' + using_px = await self.tab.evaluate(script) + if not using_px: + asyncio.create_task(self.flash_async(0.25)) asyncio.create_task( self._tab.send( cdp.input_.dispatch_mouse_event( @@ -560,12 +566,15 @@ async def mouse_click_with_offset_async( logger.debug("Clicking on location: %.2f, %.2f" % center_pos) else: logger.debug("Clicking on location: %.2f, %.2f" % (x_pos, y_pos)) - asyncio.create_task( - self.flash_async( - x_offset=x_offset - (width / 2), - y_offset=y_offset - (height / 2), - ), - ) + script = 'sessionStorage.getItem("pxsid") !== null;' + using_px = await self.tab.evaluate(script) + if not using_px: + asyncio.create_task( + self.flash_async( + x_offset=x_offset - (width / 2), + y_offset=y_offset - (height / 2), + ), + ) asyncio.create_task( self._tab.send( cdp.input_.dispatch_mouse_event( diff --git a/seleniumbase/undetected/cdp_driver/tab.py b/seleniumbase/undetected/cdp_driver/tab.py index 97ebbdf6662..1fe5dc0a834 100644 --- a/seleniumbase/undetected/cdp_driver/tab.py +++ b/seleniumbase/undetected/cdp_driver/tab.py @@ -1092,6 +1092,39 @@ async def wait_for( await self.sleep(0.068) return item + async def set_attributes(self, selector, attribute, value): + """This method uses JavaScript to set/update a common attribute. + All matching selectors from querySelectorAll() are used. + Example => (Make all links on a website redirect to Google): + self.set_attributes("a", "href", "https://google.com")""" + attribute = re.escape(attribute) + attribute = js_utils.escape_quotes_if_needed(attribute) + value = re.escape(value) + value = js_utils.escape_quotes_if_needed(value) + if selector.startswith(("/", "./", "(")): + with suppress(Exception): + selector = js_utils.convert_to_css_selector(selector, "xpath") + css_selector = selector + css_selector = re.escape(css_selector) # Add "\\" to special chars + css_selector = js_utils.escape_quotes_if_needed(css_selector) + js_code = ( + """var $elements = document.querySelectorAll('%s'); + var index = 0, length = $elements.length; + for(; index < length; index++){ + $elements[index].setAttribute('%s','%s');}""" % ( + css_selector, + attribute, + value, + ) + ) + with suppress(Exception): + await self.evaluate(js_code) + + async def internalize_links(self): + """All `target="_blank"` links become `target="_self"`. + This prevents those links from opening in a new tab.""" + await self.set_attributes('[target="_blank"]', "target", "_self") + async def download_file( self, url: str, filename: Optional[PathLike] = None ): @@ -1336,8 +1369,8 @@ async def __on_a_cf_turnstile_page(self, source=None): return True return False - async def __on_a_g_recaptcha_page(self, source=None): - await self.sleep(0.4) + async def __on_a_g_recaptcha_page(self, *args, **kwargs): + await self.sleep(0.4) # reCAPTCHA may need a moment to appear source = await self.get_html() if ( ( diff --git a/setup.py b/setup.py index ac197bf6b27..febeb93bcec 100755 --- a/setup.py +++ b/setup.py @@ -233,7 +233,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.10.7;python_version<"3.10"', - 'coverage>=7.13.0;python_version>="3.10"', + 'coverage>=7.13.1;python_version>="3.10"', 'pytest-cov>=7.0.0', ], # pip install -e .[flake8] @@ -258,7 +258,8 @@ # pip install -e .[pdfminer] # (An optional library for parsing PDF files.) "pdfminer": [ - 'pdfminer.six==20251107', + 'pdfminer.six==20251107;python_version<"3.10"', + 'pdfminer.six==20251229;python_version>="3.10"', 'cryptography==46.0.3', 'cffi==2.0.0', 'pycparser==2.23',