A site-specific selector is a CSS or HTML selection rule tailored to one website’s page structure. It can extract fields such as product titles and prices, but usually must be redesigned for another domain or layout. section 10.7
Use site-specific selectors when:
- Inspecting a product page and iterating until a selector returns the required field. section 10.7
- Extracting predictable fields from a known site with BeautifulSoup after fetching its HTML. section 10.7
- Keeping a site-specific fallback when structured-data parsing or broader extraction logic is unavailable. section 10.7
Two product sites
A product extractor can use site-specific selectors for the GORUCK GR1 page, while using structured-data parsing and a fallback strategy for the Osprey Atmos AG 65 page. Both approaches target title and price, but each reflects the source site’s HTML conventions. section 10.7
Listing 10.5 Testing extraction on two sites (title and price)
import json #A
import re #A
from pathlib import Path #B
import requests #B
from bs4 import BeautifulSoup #B
# Browser-like session headers to reduce 403 blocks
session = requests.Session() #C
session.headers.update({ #C
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/121.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Referer": "https://www.google.com/",
})
products = [ #D
("GORUCK GR1 26L", "https://www.goruck.com/products/gr1", Path("fixtures/gr1.html")),
("Osprey Atmos AG 65", "https://www.osprey.com/atmos-ag-65-atmos65s22-415?size=S%2FM&color=Mythical+Green", Path("fixtures/osprey_atmos.html")),
]
def get_html(url: str, fixture: Path) -> str:
"""Fetch HTML, fall back to a saved fixture if blocked.""" #E
try:
r = session.get(url, timeout=20, allow_redirects=True) #E
# If blocked, retry once with a couple extra "browser-ish" fetch headers
if r.status_code == 403: #E
r = session.get(
url,
timeout=20,
allow_redirects=True,
headers={
"Sec-Fetch-Site": "none",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Dest": "document",
},
)
r.raise_for_status() #E
fixture.parent.mkdir(parents=True, exist_ok=True)
fixture.write_text(r.text, encoding="utf-8") #E
return r.text
except Exception:
if fixture.exists():
print(f" Loading fixture: {fixture}") #E
return fixture.read_text(encoding="utf-8")
raise
def extract_title_price(name: str, html: str) -> dict:
soup = BeautifulSoup(html, "html.parser") #F
title_el = soup.find("h1") #F
title = title_el.get_text(" ", strip=True) if title_el else None #F
# GORUCK selector
if "GORUCK" in name: #G
price_el = soup.select_one("div.product-block__price span.price-item--regular") #G
price = price_el.get_text(" ", strip=True) if price_el else None
return {"title": title, "price": price}
# Osprey: prefer JSON-LD product schema when available
price = None
for s in soup.find_all("script", attrs={"type": "application/ld+json"}): #H
try:
data = json.loads(s.get_text(strip=True)) #H
except Exception:
continue
candidates = data if isinstance(data, list) else [data] #H
for obj in candidates:
if isinstance(obj, dict) and str(obj.get("@type", "")).lower() == "product": #H
offers = obj.get("offers", {})
if isinstance(offers, dict) and offers.get("price") is not None:
price = f"${offers['price']}"
break
if price:
break
# Fallback: regex the visible text for a dollar price
if price is None: #I
text = soup.get_text(" ", strip=True)
m = re.search(r"\$([\d,]+(?:\.\d{2})?)", text) #I
price = f"${m.group(1)}" if m else None
return {"title": title, "price": price}
for name, url, fixture in products: #J
print(f"\nFetching: {name}")
html = get_html(url, fixture)
data = extract_title_price(name, html)
print(f" Title: {data['title']}")
print(f" Price: {data['price']}")
Inspect the complete two-site extraction listing to see how fetched pages, browser-like sessions, and site-specific extraction logic fit together. section 10.7
The technique becomes difficult to maintain at scale: every new domain can require new CSS selectors, and even one site may vary selectors across variants, categories, sales states, or redesigns. section 11.1
