How to Scrape Glassdoor Company Reviews

"""
KnoxProxy scraper for public Glassdoor company overview pages.
Reads the application/ld+json structured data block for name, rating,
and review count. Does not log in or access gated review text.
"""
import json
import random
import time

import requests
from bs4 import BeautifulSoup

PROXY_HOST = "proxy.knoxproxy.com"
PROXY_PORT = 10000  # residential rotating
PROXY_USER = "your_username"
PROXY_PASS = "your_password"

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
}


def proxy_dict() -> dict:
    proxy_url = f"http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}"
    return {"http": proxy_url, "https": proxy_url}


def fetch_glassdoor_overview(employer_slug: str, max_retries: int = 3) -> dict:
    url = f"https://www.glassdoor.com/Overview/Working-at-{employer_slug}-EI.htm"

    for attempt in range(1, max_retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, proxies=proxy_dict(), timeout=20)
            if resp.status_code == 200 and "login" not in resp.url:
                return parse_glassdoor_overview(resp.text, employer_slug)
            print(f"Attempt {attempt}: hit login wall or status {resp.status_code}")
        except requests.exceptions.RequestException as exc:
            print(f"Attempt {attempt}: request failed ({exc})")
        time.sleep(random.uniform(5, 8))

    raise RuntimeError(f"Failed to fetch Glassdoor overview for {employer_slug} after {max_retries} attempts")


def parse_glassdoor_overview(html: str, employer_slug: str) -> dict:
    soup = BeautifulSoup(html, "html.parser")
    script = soup.select_one('script[type="application/ld+json"]')
    if not script or not script.string:
        return {"employer_slug": employer_slug, "error": "structured data not found"}

    data = json.loads(script.string)
    return {
        "employer_slug": employer_slug,
        "name": data.get("name"),
        "rating": data.get("aggregateRating", {}).get("ratingValue"),
        "review_count": data.get("aggregateRating", {}).get("reviewCount"),
    }


if __name__ == "__main__":
    for employer_slug in ["Example-Corp"]:
        print(fetch_glassdoor_overview(employer_slug))
        time.sleep(random.uniform(5, 8))

{
  "employer_slug": "Example-Corp",
  "name": "Example Corp",
  "rating": "3.9",
  "review_count": "2143"
}