Manual extraction creates explicit rules for retrieving selected fields from source pages. In web scraping, those rules commonly use site-specific CSS selectors, structured-data parsing, and fallback logic. The approach is straightforward for a small number of sites but accumulates special cases as domains and layouts increase. section 10.7
A practical manual extractor typically:
- Targets known HTML elements, such as a product page's h1 title and price spans. section 11.6.3
- Adds domain-specific handling when pages expose different structures, such as structured data on one site and selectors on another. section 10.7
- Maintains fallback selectors and handles missing fields, access controls, timeouts, or JavaScript-heavy rendering. section 10.7
- Compares extracted fields against another approach to assess completeness, accuracy, and implementation effort. section 11.10
A site-specific product extractor
A GORUCK extractor can parse HTML with BeautifulSoup, read the h1 element as the product name, and select either the sale-price or regular-price span. It can return null for fields without corresponding rules, such as weight or category. The same function is effective for that site's known structure, but requires additional extractors as domains multiply. section 11.6.3
Listing 10.5 Testing extraction on two sites (title and price)
import json #A
import re #A
from pathlib import Path #B
import requests #B
from bs4 import BeautifulSoup #B
# Browser-like session headers to reduce 403 blocks
session = requests.Session() #C
session.headers.update({ #C
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/121.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Referer": "https://www.google.com/",
})
products = [ #D
("GORUCK GR1 26L", "https://www.goruck.com/products/gr1", Path("fixtures/gr1.html")),
("Osprey Atmos AG 65", "https://www.osprey.com/atmos-ag-65-atmos65s22-415?size=S%2FM&color=Mythical+Green", Path("fixtures/osprey_atmos.html")),
]
def get_html(url: str, fixture: Path) -> str:
"""Fetch HTML, fall back to a saved fixture if blocked.""" #E
try:
r = session.get(url, timeout=20, allow_redirects=True) #E
# If blocked, retry once with a couple extra "browser-ish" fetch headers
if r.status_code == 403: #E
r = session.get(
url,
timeout=20,
allow_redirects=True,
headers={
"Sec-Fetch-Site": "none",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Dest": "document",
},
)
r.raise_for_status() #E
fixture.parent.mkdir(parents=True, exist_ok=True)
fixture.write_text(r.text, encoding="utf-8") #E
return r.text
except Exception:
if fixture.exists():
print(f" Loading fixture: {fixture}") #E
return fixture.read_text(encoding="utf-8")
raise
def extract_title_price(name: str, html: str) -> dict:
soup = BeautifulSoup(html, "html.parser") #F
title_el = soup.find("h1") #F
title = title_el.get_text(" ", strip=True) if title_el else None #F
# GORUCK selector
if "GORUCK" in name: #G
price_el = soup.select_one("div.product-block__price span.price-item--regular") #G
price = price_el.get_text(" ", strip=True) if price_el else None
return {"title": title, "price": price}
# Osprey: prefer JSON-LD product schema when available
price = None
for s in soup.find_all("script", attrs={"type": "application/ld+json"}): #H
try:
data = json.loads(s.get_text(strip=True)) #H
except Exception:
continue
candidates = data if isinstance(data, list) else [data] #H
for obj in candidates:
if isinstance(obj, dict) and str(obj.get("@type", "")).lower() == "product": #H
offers = obj.get("offers", {})
if isinstance(offers, dict) and offers.get("price") is not None:
price = f"${offers['price']}"
break
if price:
break
# Fallback: regex the visible text for a dollar price
if price is None: #I
text = soup.get_text(" ", strip=True)
m = re.search(r"\$([\d,]+(?:\.\d{2})?)", text) #I
price = f"${m.group(1)}" if m else None
return {"title": title, "price": price}
for name, url, fixture in products: #J
print(f"\nFetching: {name}")
html = get_html(url, fixture)
data = extract_title_price(name, html)
print(f" Title: {data['title']}")
print(f" Price: {data['price']}")
Inspect a complete two-site implementation using browser-like request headers, site-specific extraction, structured data, and fallbacks. section 10.7
