Merge pull request #4156 from seleniumbase/more-stealth-features

More Stealth Features (and more)

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -370,10 +370,15 @@ with SB(uc=True, test=True, locale="en", pls="none") as sb:
 
 ```python
 sb.cdp.get(url, **kwargs)
-sb.cdp.open(url, **kwargs)
+sb.cdp.open(url, **kwargs)  # Same as sb.cdp.get(url, **kwargs)
 sb.cdp.reload(ignore_cache=True, script_to_evaluate_on_load=None)
 sb.cdp.refresh(*args, **kwargs)
 sb.cdp.get_event_loop()
+sb.cdp.get_rd_host()  # Returns the remote-debugging host
+sb.cdp.get_rd_port()  # Returns the remote-debugging port
+sb.cdp.get_rd_url()  # Returns the remote-debugging URL
+sb.cdp.get_endpoint_url()  # Same as sb.cdp.get_rd_url()
+sb.cdp.get_port()  # Same as sb.cdp.get_rd_port()
 sb.cdp.add_handler(event, handler)
 sb.cdp.find_element(selector, best_match=False, timeout=None)
 sb.cdp.find(selector, best_match=False, timeout=None)
@@ -487,6 +492,7 @@ sb.cdp.set_attributes(selector, attribute, value)
 sb.cdp.is_attribute_present(selector, attribute, value=None)
 sb.cdp.is_online()
 sb.cdp.solve_captcha()
+sb.cdp.click_captcha()
 sb.cdp.gui_press_key(key)
 sb.cdp.gui_press_keys(keys)
 sb.cdp.gui_write(text)
@@ -612,6 +618,69 @@ sb.driver.stop()
 
 --------
 
+### 🐙 <b translate="no">CDP Mode</b> Async API / Methods
+
+```python
+await get(url="about:blank")
+await open(url="about:blank")
+await find(text, best_match=False, timeout=10)  # `text` can be a selector
+await find_all(text, timeout=10)  # `text` can be a selector
+await select(selector, timeout=10)
+await select_all(selector, timeout=10, include_frames=False)
+await query_selector(selector)
+await query_selector_all(selector)
+await find_element_by_text(text, best_match=False)
+await find_elements_by_text(text)
+await reload(ignore_cache=True, script_to_evaluate_on_load=None)
+await evaluate(expression)
+await js_dumps(obj_name)
+await back()
+await forward()
+await get_window()
+await get_content()
+await maximize()
+await minimize()
+await fullscreen()
+await medimize()
+await set_window_size(left=0, top=0, width=1280, height=1024)
+await set_window_rect(left=0, top=0, width=1280, height=1024)
+await activate()
+await bring_to_front()
+await set_window_state(left=0, top=0, width=1280, height=720, state="normal")
+await get_navigation_history()
+await open_external_inspector()  # Open a separate browser for debugging
+await close()
+await scroll_down(amount=25)
+await scroll_up(amount=25)
+await wait_for(selector="", text="", timeout=10)
+await download_file(url, filename=None)
+await save_screenshot(filename="auto", format="png", full_page=False)
+await print_to_pdf(filename="auto")
+await set_download_path(path)
+await get_all_linked_sources()
+await get_all_urls(absolute=True)
+await get_html()
+await get_page_source()
+await is_element_present(selector)
+await is_element_visible(selector)
+await get_element_rect(selector, timeout=5)  # (relative to window)
+await get_window_rect()
+await get_gui_element_rect(selector, timeout=5)  # (relative to screen)
+await get_title()
+await send_keys(selector, text, timeout=5)
+await type(selector, text, timeout=5)
+await click(selector, timeout=5)
+await click_with_offset(selector, x, y, center=False, timeout=5)
+await solve_captcha()
+await click_captcha()  # Same as solve_captcha()
+await get_document()
+await get_flattened_document()
+await get_local_storage()
+await set_local_storage(items)
+```
+
+--------
+
 ### 🐙 <b translate="no">CDP Mode</b> WebElement API / Methods
 
 After finding an element in CDP Mode, you can access `WebElement` methods:

examples/cdp_mode/playwright/ReadMe.md

@@ -0,0 +1,141 @@
+<!-- SeleniumBase Docs -->
+
+<h2><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/img/logo6.png" title="SeleniumBase" width="32"></a> Stealthy Playwright 🎭</h2>
+
+🎭 <b translate="no">Stealthy Playwright Mode</b> is a special mode of <b translate="no">SeleniumBase</b> that launches <a href="https://github.com/microsoft/playwright-python">Playwright</a> from <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md" translate="no">SeleniumBase CDP Mode</a> in order to grant <b translate="no">Playwright</b> new stealth features, such as the ability to click CAPTCHA checkboxes successfully.  <b translate="no">Playwright</b> uses <code>connect_over_cdp()</code> to attach itself onto an existing <b translate="no">SeleniumBase</b> session via the <code>remote-debugging-port</code>. From here, APIs of both frameworks can be used, giving you a hybrid approach that delivers the best experience of both worlds.
+
+--------
+
+### 🎭 Getting started with <b translate="no">Stealthy Playwright Mode</b>:
+
+If `playwright` isn't already installed, then install it first:
+
+```zsh
+pip install playwright
+```
+
+Stealthy Playwright Mode comes in 3 formats:
+1. `sb_cdp` sync format
+2. `SB` nested sync format
+3. `cdp_driver` async format
+
+
+#### `sb_cdp` sync format (minimal boilerplate):
+
+```python
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    page.goto("https://example.com")
+```
+
+#### `SB` nested sync format (minimal boilerplate):
+
+```python
+from playwright.sync_api import sync_playwright
+from seleniumbase import SB
+
+with SB(uc=True) as sb:
+    sb.activate_cdp_mode()
+    endpoint_url = sb.cdp.get_endpoint_url()
+
+    with sync_playwright() as p:
+        browser = p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        page.goto("https://example.com")
+```
+
+#### `cdp_driver` async format (minimal boilerplate):
+
+```python
+import asyncio
+from seleniumbase import cdp_driver
+from playwright.async_api import async_playwright
+
+async def main():
+    driver = await cdp_driver.start_async()
+    endpoint_url = driver.get_endpoint_url()
+
+    async with async_playwright() as p:
+        browser = await p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        await page.goto("https://example.com")
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())
+```
+
+### 🎭 <b translate="no">Stealthy Playwright Mode</b> details:
+
+The `sb_cdp` and `cdp_driver` formats don't use WebDriver at all, meaning that `chromedriver` isn't needed. From these two formats, Stealthy Playwright Mode can call [CDP Mode methods](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/cdp_mode_methods.md) and Playwright methods.
+
+The `SB()` format requires WebDriver, therefore `chromedriver` will be downloaded (as `uc_driver`) if the driver isn't already present on the local machine. The `SB()` format has access to Selenium WebDriver methods via [the SeleniumBase API](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/method_summary.md). Using Stealthy Playwright Mode from `SB()` grants access to all the APIs: Selenium, SeleniumBase, [UC Mode](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/uc_mode.md), [CDP Mode](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md), and Playwright.
+
+In the sync formats, `get_endpoint_url()` also applies `nest-asyncio` so that nested event loops are allowed. (Python doesn't allow nested event loops by default). Without this, you'd get the error: `"Cannot run the event loop while another loop is running"` when calling CDP Mode methods (such as `solve_captcha()`) from within the Playwright context manager. This `nest-asyncio` call is done behind-the-scenes so that users don't need to handle this on their own.
+
+### 🎭 <b translate="no">Stealthy Playwright Mode</b> examples:
+
+Here's an example that queries Microsoft Copilot:
+
+```python
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    page.goto("https://copilot.microsoft.com")
+    page.wait_for_selector("textarea#userInput")
+    sb.sleep(1)
+    query = "Playwright Python connect_over_cdp() sync example"
+    page.fill("textarea#userInput", query)
+    page.click('button[data-testid="submit-button"]')
+    sb.sleep(3)
+    sb.solve_captcha()
+    page.wait_for_selector('button[data-testid*="-thumbs-up"]')
+    sb.sleep(4)
+    page.click('button[data-testid*="scroll-to-bottom"]')
+    sb.sleep(3)
+    chat_results = '[data-testid="highlighted-chats"]'
+    result = page.locator(chat_results).inner_text()
+    print(result.replace("\n\n", " \n"))
+```
+
+Here's an example that solves the Bing CAPTCHA:
+
+```python
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome(locale="en")
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    page.goto("https://www.bing.com/turing/captcha/challenge")
+    sb.sleep(3)
+    sb.solve_captcha()
+    sb.sleep(3)
+```
+
+For more examples, see [examples/cdp_mode/playwright](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/playwright).
+
+--------
+
+<a href="https://github.com/seleniumbase/SeleniumBase"><img src="https://seleniumbase.github.io/img/logo6.png" alt="SeleniumBase" title="SeleniumBase" width="100" /></a><img src="https://seleniumbase.github.io/other/playwright_logo.png" alt="Playwright" title="SeleniumBase" height="100">

examples/cdp_mode/playwright/__init__.py

@@ -0,0 +1,24 @@
+import asyncio
+from playwright.async_api import async_playwright
+from seleniumbase import cdp_driver
+
+
+async def main():
+    driver = await cdp_driver.start_async()
+    endpoint_url = driver.get_endpoint_url()
+
+    async with async_playwright() as p:
+        browser = await p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        await page.goto("https://seleniumbase.io/simple/login")
+        await page.fill("#username", "demo_user")
+        await page.fill("#password", "secret_pass")
+        await page.click("#log-in")
+        await page.wait_for_selector("h1")
+        await driver.sleep(1)
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())

examples/cdp_mode/playwright/raw_basic_async.py

@@ -0,0 +1,17 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import SB
+
+with SB(uc=True) as sb:
+    sb.activate_cdp_mode()
+    endpoint_url = sb.cdp.get_endpoint_url()
+
+    with sync_playwright() as p:
+        browser = p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        page.goto("https://seleniumbase.io/simple/login")
+        page.fill("#username", "demo_user")
+        page.fill("#password", "secret_pass")
+        page.click("#log-in")
+        page.wait_for_selector("h1")
+        sb.sleep(1)

examples/cdp_mode/playwright/raw_basic_nested.py

@@ -0,0 +1,16 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome()
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    page.goto("https://seleniumbase.io/simple/login")
+    page.fill("#username", "demo_user")
+    page.fill("#password", "secret_pass")
+    page.click("#log-in")
+    page.wait_for_selector("h1")
+    sb.sleep(1)

examples/cdp_mode/playwright/raw_basic_sync.py

@@ -0,0 +1,22 @@
+import asyncio
+from playwright.async_api import async_playwright
+from seleniumbase import cdp_driver
+
+
+async def main():
+    driver = await cdp_driver.start_async(locale="en")
+    endpoint_url = driver.get_endpoint_url()
+
+    async with async_playwright() as p:
+        browser = await p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        await page.goto("https://www.bing.com/turing/captcha/challenge")
+        await driver.sleep(3)
+        await driver.solve_captcha()
+        await driver.sleep(3)
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())

examples/cdp_mode/playwright/raw_bing_cap_async.py

@@ -0,0 +1,15 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import SB
+
+with SB(uc=True, locale="en") as sb:
+    sb.activate_cdp_mode()
+    endpoint_url = sb.cdp.get_endpoint_url()
+
+    with sync_playwright() as p:
+        browser = p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        page.goto("https://www.bing.com/turing/captcha/challenge")
+        sb.sleep(3)
+        sb.solve_captcha()
+        sb.sleep(3)

examples/cdp_mode/playwright/raw_bing_cap_nested.py

@@ -0,0 +1,14 @@
+from playwright.sync_api import sync_playwright
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome(locale="en")
+endpoint_url = sb.get_endpoint_url()
+
+with sync_playwright() as p:
+    browser = p.chromium.connect_over_cdp(endpoint_url)
+    context = browser.contexts[0]
+    page = context.pages[0]
+    page.goto("https://www.bing.com/turing/captcha/challenge")
+    sb.sleep(3)
+    sb.solve_captcha()
+    sb.sleep(3)

examples/cdp_mode/playwright/raw_bing_cap_sync.py

@@ -0,0 +1,33 @@
+import asyncio
+from playwright.async_api import async_playwright
+from seleniumbase import cdp_driver
+
+
+async def main():
+    driver = await cdp_driver.start_async()
+    endpoint_url = driver.get_endpoint_url()
+
+    async with async_playwright() as p:
+        browser = await p.chromium.connect_over_cdp(endpoint_url)
+        context = browser.contexts[0]
+        page = context.pages[0]
+        await page.goto("https://copilot.microsoft.com")
+        await page.wait_for_selector("textarea#userInput")
+        await driver.sleep(1)
+        query = "Playwright Python connect_over_cdp() sync example"
+        await page.fill("textarea#userInput", query)
+        await page.click('button[data-testid="submit-button"]')
+        await driver.sleep(3)
+        await driver.solve_captcha()
+        await page.wait_for_selector('button[data-testid*="-thumbs-up"]')
+        await driver.sleep(4)
+        await page.click('button[data-testid*="scroll-to-bottom"]')
+        await driver.sleep(3)
+        chat_results = '[data-testid="highlighted-chats"]'
+        result = await page.locator(chat_results).inner_text()
+        print(result.replace("\n\n", " \n"))
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())