Selenium WebDriver

Automate browser interactions for dynamic web scraping.

Installation

pip install selenium webdriver-manager

Quick Start

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager

# Setup headless Chrome
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)

try:
    driver.get('https://example.com')
    element = driver.find_element(By.CSS_SELECTOR, 'h1')
    print(element.text)
finally:
    driver.quit()

Finding Elements

from selenium.webdriver.common.by import By

# By ID
driver.find_element(By.ID, 'main')

# By class name
driver.find_elements(By.CLASS_NAME, 'item')

# By CSS selector
driver.find_element(By.CSS_SELECTOR, 'div.content > p')

# By XPath
driver.find_element(By.XPATH, '//button[@type="submit"]')

# By link text
driver.find_element(By.LINK_TEXT, 'Click here')

Waiting for Elements

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Wait up to 10 seconds for element
wait = WebDriverWait(driver, 10)
element = wait.until(
    EC.presence_of_element_located((By.ID, 'dynamic-content'))
)

# Wait for element to be clickable
button = wait.until(
    EC.element_to_be_clickable((By.CSS_SELECTOR, 'button.submit'))
)

Interactions

# Click
driver.find_element(By.CSS_SELECTOR, 'button').click()

# Type text
input_field = driver.find_element(By.NAME, 'search')
input_field.clear()
input_field.send_keys('query')

# Submit form
form = driver.find_element(By.TAG_NAME, 'form')
form.submit()

# Scroll
driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')

When to Use Selenium

• JavaScript-rendered content
• Single-page applications (SPAs)
• Form submissions and authentication
• Content behind login walls
• Interactive elements (dropdowns, modals)

When NOT to Use Selenium

• Static HTML pages (use requests + BeautifulSoup)
• API endpoints returning JSON
• Already downloaded HTML files
• Simple HTTP requests