How to Scrape Google Flights With Python: A Step-by-Step Tutorial

				
					from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver import Keys, ActionChains
import csv

start_url = "https://www.google.com/travel/flights"
timeout = 10

my_params = {
    'from': 'LAX',
    'to': 'Atlanta',
    'departure': '2024-11-27',
    'return': '2024-12-03'
}
my_params2 = {
    'from': 'LAX',
    'to': 'Atlanta',
    'departure': '2024-11-27',
    'return': None
}

def prepare_browser() -> webdriver:
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=chrome_options)
    return driver

def field_input(driver, element, text) -> None:
    element.click()
    ActionChains(driver)\
    .move_to_element(element)\
    .send_keys(text)\
    .pause(1)\
    .send_keys(Keys.ENTER)\
    .perform()

def prepare_query(driver) -> None:
    field_from = driver.find_element(By.CSS_SELECTOR, 'div.BGeFcf div[jsname="pT3pqd"] input')
    field_to = driver.find_element(By.CSS_SELECTOR, 'div.vxNK6d div[jsname="pT3pqd"] input')
    date_from = driver.find_element(By.CSS_SELECTOR, 'div[jsname="I3Yihd"] input')
    date_to = None
    if my_params['return'] is None or my_params['return'] == '':
        dropdown = driver.find_element(By.CSS_SELECTOR, 'div.VfPpkd-aPP78e')
        dropdown.click()
        ActionChains(driver)\
            .move_to_element(dropdown)\
            .send_keys(Keys.ARROW_DOWN)\
            .pause(1)\
            .send_keys(Keys.ENTER)\
            .perform()
    else:
        date_to = driver.find_element(By.CSS_SELECTOR, 'div.K2bCpe div[jsname="CpWD9d"] input')
    field_input(driver, field_from, my_params['from'])
    field_input(driver, field_to, my_params['to'])
    field_input(driver, date_from, my_params['departure'])
    if date_to is not None:
        field_input(driver, date_to, my_params['return'])
    ActionChains(driver)\
        .send_keys(Keys.ENTER)\
        .perform()
    print("Done preparing the search query")

def get_flight_info(element, best) -> dict:
    times = element.find_elements(By.CSS_SELECTOR, 'div.Ir0Voe span[role="text"]')
    airports = element.find_elements(By.CSS_SELECTOR, 'div.QylvBf span span[jscontroller="cNtv4b"]')
    flight_info = {
        'airline': element.find_element(By.CSS_SELECTOR, 'div.Ir0Voe > div.sSHqwe.tPgKwe.ogfYpf > span:last-child').text,
        'date_leave': my_params['departure'],
        'date_arrive': my_params['return'],
        'time_leave': times[0].text,
        'time_arrive': times[1].text,
        'duration_string': element.find_element(By.CSS_SELECTOR, 'div.Ak5kof > div.gvkrdb.AdWm1c.tPgKwe.ogfYpf').text,
        'airport_leave': airports[0].text,
        'airport_arrive': airports[1].text,
        'layovers': element.find_element(By.CSS_SELECTOR, 'div.BbR8Ec > div.EfT7Ae.AdWm1c.tPgKwe > span').text,
        'cost': element.find_element(By.CSS_SELECTOR, 'div.U3gSDe div div.YMlIz.FpEdX span').text,
        'best': best
    }
    return flight_info

def parse(list_elem: list, best: bool) -> list:
    list_items = list_elem.find_elements(By.CSS_SELECTOR, 'li.pIav2d')
    list_of_flights = []
    for list_item in list_items:
        list_of_flights.append(get_flight_info(list_item, best))
    return list_of_flights

def find_lists(driver):
    list_elems = driver.find_elements(By.CSS_SELECTOR, 'ul.Rk10dc')
    list_of_flights = parse(list_elems[0], True) + parse(list_elems[1], False)
    return list_of_flights

def write_to_csv(flights):
    field_names = ['airline', 'date_leave', 'date_arrive', 'time_leave',
                   'time_arrive', 'duration_string', 'airport_leave',
                   'airport_arrive', 'layovers', 'cost', 'best']
    output_filename = 'flights.csv'
    with open(output_filename, 'w', newline='', encoding='utf-8') as f_out:
        writer = csv.DictWriter(f_out, fieldnames=field_names)
        writer.writeheader()
        writer.writerows(flights)

def scrape(driver: webdriver) -> None:
    driver.get(start_url)
    if driver.current_url.startswith("https://consent.google.com"):
        print("Hit the consent page, dealing with it.")
        btn_consent_allow = driver.find_element(By.CSS_SELECTOR, 'button.VfPpkd-LgbsSe[jsname="b3VHJd')
        btn_consent_allow.click()
        try:
            WebDriverWait(driver, timeout).until(
                expected_conditions.presence_of_element_located((
                    By.CSS_SELECTOR, 'button[jsname="vLv7Lb"]'))
            )
            print("Search button found, continuing.")
            prepare_query(driver)
        except Exception as e:
            print("Something went wrong: {e}")
        search_btn = driver.find_element(By.CSS_SELECTOR, 'button[jsname="vLv7Lb"]')
        search_btn.click()
        try:
            WebDriverWait(driver, timeout).until(
                expected_conditions.presence_of_element_located((
                    By.CSS_SELECTOR, 'ul.Rk10dc'))
            )
            print("Got the results back.")
            flights = find_lists(driver)
            write_to_csv(flights)
        except Exception as e:
            print(f"Something went wrong: {e}")

def main() -> None:
    driver = prepare_browser()
    scrape(driver)
    driver.quit()

if __name__ == '__main__':
    main()