How to Scrape Amazon Product Data

"""
KnoxProxy scraper for Amazon product pages.
Extracts title, price, rating, review count, and availability from a
public Amazon product detail page (ASIN-based URL).
"""
import random
import time

import requests
from bs4 import BeautifulSoup

PROXY_HOST = "proxy.knoxproxy.com"
PROXY_PORT = 10000  # residential rotating
PROXY_USER = "your_username"
PROXY_PASS = "your_password"

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}


def build_proxy_url(country: str = "us") -> str:
    user = f"{PROXY_USER}-country-{country}"
    return f"http://{user}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"


def fetch_amazon_product(asin: str, country: str = "us", max_retries: int = 3) -> dict:
    url = f"https://www.amazon.com/dp/{asin}"
    proxy_url = build_proxy_url(country)
    proxies = {"http": proxy_url, "https": proxy_url}

    for attempt in range(1, max_retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, proxies=proxies, timeout=20)
            if resp.status_code == 200 and "captcha" not in resp.text.lower():
                return parse_amazon_page(resp.text, asin)
            print(f"Attempt {attempt}: status {resp.status_code}, retrying with a new IP")
        except requests.exceptions.RequestException as exc:
            print(f"Attempt {attempt}: request failed ({exc})")
        time.sleep(random.uniform(2, 5))

    raise RuntimeError(f"Failed to fetch Amazon ASIN {asin} after {max_retries} attempts")


def parse_amazon_page(html: str, asin: str) -> dict:
    soup = BeautifulSoup(html, "html.parser")
    title_el = soup.select_one("#productTitle")
    price_el = soup.select_one(".a-price .a-offscreen")
    rating_el = soup.select_one("span.a-icon-alt")
    review_count_el = soup.select_one("#acrCustomerReviewText")
    availability_el = soup.select_one("#availability span")

    return {
        "asin": asin,
        "title": title_el.get_text(strip=True) if title_el else None,
        "price": price_el.get_text(strip=True) if price_el else None,
        "rating": rating_el.get_text(strip=True) if rating_el else None,
        "review_count": review_count_el.get_text(strip=True) if review_count_el else None,
        "availability": availability_el.get_text(strip=True) if availability_el else None,
    }


if __name__ == "__main__":
    asins = ["B0BSHF7WHW", "B08N5WRWNW"]
    for asin in asins:
        print(fetch_amazon_product(asin))
        time.sleep(random.uniform(2, 5))  # pace requests between ASINs

{
  "asin": "B0BSHF7WHW",
  "title": "Echo Dot (5th Gen) Smart Speaker with Alexa",
  "price": "$29.99",
  "rating": "4.7 out of 5 stars",
  "review_count": "89,412",
  "availability": "In Stock"
}