Cheerio Web Scraping: A Tutorial on How to Scrape Static Pages with Node.js 

				
					import axios from 'axios'
import { load } from 'cheerio'

// For writing into the output file
import fs from 'fs'

const start_url = "http://books.toscrape.com/"

const books_list = []

    // Requesting the page with the help of Axios and waiting for the response
    let resp = await axios.get(url)
    let resp_html = resp.data
    // Loading the html into Cheerio. $ - Cheerio object
    const $ = load(resp_html)
    // Passing the Cheerio instance to the parse() function
    parse($)

    try {
        // Try finding the  next page selector and
        // extract the href attribute for scraping the next page
        let next_href = $('.next > a').attr("href")
        // In case the '/catalogue/' part of the URL is not found within
        // the href attribute value, add it to the href
        if (!next_href.includes('catalogue')){
            next_href = `catalogue/${next_href}`
        }
        // Formatting the absolute URL of the next page we are going to scrape
        let next_url = start_url + next_href
        console.log('Scrape: ' + next_url)
        // Calling the scrape() function again and passing it the URL
        await scrape(next_url)

    } catch {
        // Next page selector not found, end job
        return
    }

}

// Function for parsing the html of the page.
function parse($){
    // The selector for each distinct book element on the page is an article
    // tag with the class of "product_pod". This line finds all such elements
    // and begins iterating through them.
    $('.product_pod').map((i, element) => {
        // To get the title, we find the h3 tag within the element and
        // extract its text.
        const book_title = $(element).find('h3').text()
        // Price is also simple, we just get rid of the pound sign
        const book_price = $(element).find('.price_color').text().replace('£', '')
        // The book ratings are easily scraped from the p tag with the classes
        // "star rating" and "Num" where "Num" is the rating the book has
        // received. To extract the rating, we first find the element with that
        // class, get the value of the "class" attribute which returns a string:
        // e.g. "star-rating One", split that string by whitespaces and assign
        // the second element of the resulting list to our variable.
        const book_rating = $(element).find('p.star-rating').attr("class")
            .split(' ')[1]
        // Simply finding the element by the "instock" class, extracting the
        // text and trimming the resulting string to strip away unnecessary
        // whitespaces.
        const book_stock = $(element).find('.instock').text().trim()
        // To extract the url of the book, we find the a tag within the
        // product_pod element and get its "href" attribute which we append to
        // the start_url
        const book_url = start_url + $(element).find('a').attr("href")
        // Appending the results dictionary to the books_list
        books_list.push({
            "title": book_title,
            "price": book_price,
            "rating": book_rating,
            "stock": book_stock,
            "url": book_url
        })
    })
    //console.log(books)
}

function write_to_csv(){
    // Getting the keys from the books object, this will become the first line of the csv file
    var csv = Object.keys(books_list[0]).join(', ') + '\n'
    // Iterating through each book dictionary element
    books_list.forEach(function(book) {
        // Adding a new line to the csv variable with the line break at the end
        csv += `"${book['title']}", ${book['price']}, ${book['rating']}, ${book['stock']}, ${book['url']},\n`
    })
    //console.log(csv)
    // Writing the output to a output.csv file
    fs.writeFile('output.csv', csv, (err) => {
        if (err)
            console.log(err)
        else {
            console.log("Output written successfully")
        }
    })
}

// Script starts here, we pass the URL we are going to start our scrape on to
// the scrape function and tell node for it to be awaited so that all of the
// scrapes finish before we move on to writing the output
await scrape(start_url)
// Calling the function to write the output
write_to_csv()