How to Scrape Reddit with Python: an Easy Web Scraping Tutorial

				
					import requests
from bs4 import BeautifulSoup
from datetime import datetime

start_url = "https://old.reddit.com/r/playstation/"
# to scrape new: https://old.reddit.com/r/playstation/new/

proxy = "server:port"

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "en-US,en;q=0.9",
    "Referer": "https://old.reddit.com/",
    "Sec-Ch-Ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"", 
    "Sec-Ch-Ua-Mobile": "?0", 
    "Sec-Ch-Ua-Platform": "\"Windows\"", 
    "Sec-Fetch-Dest": "document", 
    "Sec-Fetch-Mode": "navigate", 
    "Sec-Fetch-Site": "cross-site", 
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
}

# Number of pages to scrape.
num_pages = 3

def parse_page(content: str, page: int):
    # Parsing the page content with BeautifulSoup.
    soup = BeautifulSoup(content, "html.parser")
    # Finding all divs that have a class "thing".
    items = soup.find_all("div", class_="thing")
    # Preparing a list that will hold a list of dictionaries.
    scraped_items = []
    # Printing out the amount found in the page.
    print (f"Items: {len(items)}")
    # Iterating through each found item (div element).
    for item in items:
        # We can create a dictionary item_info and immediately add information.
        item_info = {
            "rank": item.attrs["data-rank"],
            "score": item.attrs["data-score"],
            "author": item.attrs["data-author"],
            "promoted": item.attrs["data-promoted"],
            "subreddit": item.attrs["data-subreddit"],
            "timestamp": int(item.attrs["data-timestamp"]),
            "url": item.attrs["data-url"],
            "comments": item.attrs["data-comments-count"],
            "nsfw": item.attrs["data-nsfw"],
            "crossposts": item.attrs["data-num-crossposts"],
            "domain": item.attrs["data-domain"],
            "page": page,
        }
        # The timestamp is in milliseconds. Here we convert it to seconds and
        # formatting it into a human readable string.
        item_info["datetime"] = datetime.fromtimestamp \
            (item_info["timestamp"] / 1000).strftime("%m/%d/%Y, %H:%M:%S")
        # The title is contained within a 
 tag with a "title" class. 
        # Here we use a BeautifoulSoup .find() function to locate the element
        # within the item div and extract its text.
        item_info["title"] = item.find("p", class_="title").get_text()
        # Now just append the resulting dictionary to the scraped_items list.
        scraped_items.append(item_info)
    # After parsing all of the items, we can get the URL of the next page to 
    # scrape. We can find it by finding the span tag with the class of 
    # "next-button" and navigating down the parse tree to the <a> child element
    # and extracting its "href" attribute.
    next_url = soup.find("span", class_="next-button").a.attrs["href"]
    print (f"Next URL: {next_url}")
    # Finally, we return the list of scraped items as well as the url to the
    # next page.
    return scraped_items, next_url

def scrape_page(url: str, page_num: int, scraped_content: list):

    if page_num <= num_pages:
        try:
            resp = requests.get(
                url,
                proxies={'http':proxy, 'https':proxy},
                headers=headers)
            print (f"Response status: {resp.status_code}")
            resp_content = resp.text
            # Calling the parse_page() function to return a  list containing
            # dictionaries of parsed content and the URL of the next page to
            # scrape. 
            scraped_items, next_url = parse_page(resp_content, page_num)
            # Returned scraped items are added to the scraped_content list.
            scraped_content.extend(scraped_items)
            next_page_num = page_num + 1
            # Calling the scrape_page() functio again to scrape the next page
            # now that we have its URL.
            scrape_page(next_url, next_page_num, scraped_content)
        except Exception as exc:
            # In case something happens and the request fails, print the 
            # exception.
            # Retry functionality can be implemented here.
            print (exc)

def main():
    # Starting on page 1.
    page = 1
    # List for storing all of the scraped output.
    scraped_content = []
    # Calling the scrape_page() function to begin work.
    scrape_page(start_url, page, scraped_content)
    # Printing out the output after scraping is finished.
    print (scraped_content)

if __name__ == "__main__":
    main()