Selenium

From Code Self Study Wiki
Jump to: navigation, search

Web browser automation.

Installation[edit]

pip install selenium

Usage[edit]

from selenium import webdriver
import mongoengine as m
from models import save_paragraph
 
url = 'http://example.com/'
 
DB_NAME = 'sample_db'
# Creates a Mongo database, if it doesn't exist.
m.connect(DB_NAME)
 
browser = webdriver.Firefox() # or webdriver.PhantomJS() for a headless browser
browser.get(url)
 
# Scrape some data from the page
page_tile = browser.title
paragraphs = browser.find_elements_by_tag_name('p')
 
# Do something with the data
for paragraph in paragraphs:
    save_paragraph(page_title, url, paragraph.text)