A Requests session persists shared settings and reuses connections across multiple HTTP requests. For multi-page scraping, configure browser-like headers once instead of repeating them on every call. This can improve performance and consistency and may avoid simple 403 blocks, although it does not guarantee access. section 10.7
Listing 10.5 Testing extraction on two sites (title and price)
import json #A
import re #A
from pathlib import Path #B
import requests #B
from bs4 import BeautifulSoup #B
# Browser-like session headers to reduce 403 blocks
session = requests.Session() #C
session.headers.update({ #C
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/121.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Referer": "https://www.google.com/",
})
products = [ #D
("GORUCK GR1 26L", "https://www.goruck.com/products/gr1", Path("fixtures/gr1.html")),
("Osprey Atmos AG 65", "https://www.osprey.com/atmos-ag-65-atmos65s22-415?size=S%2FM&color=Mythical+Green", Path("fixtures/osprey_atmos.html")),
]
def get_html(url: str, fixture: Path) -> str:
"""Fetch HTML, fall back to a saved fixture if blocked.""" #E
try:
r = session.get(url, timeout=20, allow_redirects=True) #E
# If blocked, retry once with a couple extra "browser-ish" fetch headers
if r.status_code == 403: #E
r = session.get(
url,
timeout=20,
allow_redirects=True,
headers={
"Sec-Fetch-Site": "none",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Dest": "document",
},
)
r.raise_for_status() #E
fixture.parent.mkdir(parents=True, exist_ok=True)
fixture.write_text(r.text, encoding="utf-8") #E
return r.text
except Exception:
if fixture.exists():
print(f" Loading fixture: {fixture}") #E
return fixture.read_text(encoding="utf-8")
raise
def extract_title_price(name: str, html: str) -> dict:
soup = BeautifulSoup(html, "html.parser") #F
title_el = soup.find("h1") #F
title = title_el.get_text(" ", strip=True) if title_el else None #F
# GORUCK selector
if "GORUCK" in name: #G
price_el = soup.select_one("div.product-block__price span.price-item--regular") #G
price = price_el.get_text(" ", strip=True) if price_el else None
return {"title": title, "price": price}
# Osprey: prefer JSON-LD product schema when available
price = None
for s in soup.find_all("script", attrs={"type": "application/ld+json"}): #H
try:
data = json.loads(s.get_text(strip=True)) #H
except Exception:
continue
candidates = data if isinstance(data, list) else [data] #H
for obj in candidates:
if isinstance(obj, dict) and str(obj.get("@type", "")).lower() == "product": #H
offers = obj.get("offers", {})
if isinstance(offers, dict) and offers.get("price") is not None:
price = f"${offers['price']}"
break
if price:
break
# Fallback: regex the visible text for a dollar price
if price is None: #I
text = soup.get_text(" ", strip=True)
m = re.search(r"\$([\d,]+(?:\.\d{2})?)", text) #I
price = f"${m.group(1)}" if m else None
return {"title": title, "price": price}
for name, url, fixture in products: #J
print(f"\nFetching: {name}")
html = get_html(url, fixture)
data = extract_title_price(name, html)
print(f" Title: {data['title']}")
print(f" Price: {data['price']}")
A complete extraction harness creates one session, applies browser-like headers, and uses it for requests to two product pages. section 10.7
A reusable session supports a more reliable scraping workflow:
- Create a requests.Session() object and update its headers with at least a realistic User-Agent; Accept, Accept-Language, and Referer can also be included. section 10.8
- Centralize fetching in a helper that calls session.get(), supplies a timeout, invokes raise_for_status(), and returns response.text. section 10.8
- Reuse the session while combining it with BeautifulSoup extraction and site-specific or fallback parsing strategies. section 10.7
