I have a working test script you can try it by just copy and paste, and it failed with stale element error incorporated it into a loop. I am looping through google :"gasbuddy + “addresses in inputfile” and click the first link and redirect me to that webpage. It reports “stale element no found” after successfully sent the keys and I see the results loaded in driver. Due to data buffering issue, time.sleep is not working here, so I used webdriverwait, I think this is the only difference between this failed script and my test scrip which works. I attached both scripts here, what is happening to these a[h3] wraps that have all the google search results, why it disappeared from DOM?
Test script which works perfectly:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
import re
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# Setup Selenium WebDriver
service = Service("C:/Users/ywang/.spyder-py3/chromedriver.exe")
driver = webdriver.Chrome(service=service)
# Ensure the chromedriver is in your PATH or specify the path to it
# List of gas stations
gas_stations = [
{"1037 SAINT JAMES AVE SPRINGFIELD, MA 01104-1309"
},
# Add more gas stations here
]
# Loop through each gas station
for station in gas_stations:
search_query = f"gasbuddy {station}"
# Open Google
driver.get("https://www.google.com")
search_box = driver.find_element(By.NAME, "q")
search_box.send_keys(search_query)
search_box.send_keys(Keys.RETURN)
#time.sleep(5) # Wait for search results to load
wait = WebDriverWait(driver, 5)
wait.until(EC.presence_of_element_located((By.XPATH, "//a[h3]")))
# Find GasBuddy link in search results
links = driver.find_elements(By.XPATH, "//a[h3]") #finds all anchor (<a>) elements on the Google search results page that contain a child <h3> element. These are typically the clickable titles of the search results.
gasbuddy_link = None
for link in links:
if "gasbuddy.com" in link.get_attribute("href"):
gasbuddy_link = link.get_attribute("href")
break
if gasbuddy_link:
driver.get(gasbuddy_link)
time.sleep(3) # Wait for the GasBuddy page to load
# Extract station ID from the URL
station_id_match = re.search(r'/station/(\d+)', gasbuddy_link)
station_id = station_id_match.group(1) if station_id_match else 'N/A'
# Scrape ratings (this part will depend on the structure of the GasBuddy page)
try:
rating = driver.find_element(By.XPATH, ".//span[contains(@class, 'Station-module__ratingAverage')]").text
ratingcount = driver.find_element(By.XPATH, "//span[contains(@class, 'StationInfoBox-module__ratings___1O33R')]").text.strip("()")
amenities_elements = driver.find_elements(By.XPATH, "//div[contains(@class, 'amenitiesPanel')]//span[contains(@class, 'text__left___1iOw3')]")
amenities = [element.text.strip() for element in amenities_elements if element.text.strip()]
#amenities = [element.text for element in amenities_elements]
num_amenities = len(amenities)
amenities_str = ', '.join(amenities)
except Exception as e:
print(f"Failed to get rating for {station['name']}: {e}")
print(f"GasBuddy Link: {gasbuddy_link}")
print(f"Station ID: {station_id}")
print(f"Station Rating: {rating}")
print(f"Amount of Ratings: {ratingcount}")
# Print or process the list of amenities
print(f"Number of amenities: {num_amenities}")
print(f"Amenities: {amenities_str}")
else:
print(f"No GasBuddy link found for {station['name']}")
time.sleep(2) # Small delay before the next iteration
# Close the driver
driver.quit()
Below is the problematic part of the script inside a loop (it reports the error code after the line sent the keys). I can not use time.sleep, otherwise I suspect it might work. I used wait.until(driver.find_elements(By.XPATH, "//a[h3]")) and I used wait.until(lambda driver: len(driver.find_elements(By.XPATH, "//a[h3]")) >= 10) and adjusted how many links to load before stop to control the waiting time. I waited for the page load completed: wait.until(lambda driver: driver.execute_script('return document.readyState') == 'complete') , it is still the same error. My guess is that when the google is loading all the results/links, the elements are constantly changing. Therefore, as long as I do not let the page to fully load and stop, this error will appear? But it does not make sense that the test script works. Where is the problem?
def scrape_stationinfo(log_file_name, address_chunk, output_file, proxies):
logging.basicConfig(filename=log_file_name, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
proxy_cycle = cycle(proxies) #function to cycle through incoming selected proxies
chrome_driver_path = 'C:/Users/ywang/.spyder-py3/chromedriver.exe'
service = Service(chrome_driver_path)
#driver = None
try:
# Loop through each address
for index, row in address_chunk.iterrows():
address = row['Address']
proxy_success = False
driver = None
for _ in range(len(proxies)):
proxy = next(proxy_cycle)
try:
if driver is not None:
driver.close()
driver.quit()
# Open the website using the current proxy
chrome_options = Options()
#chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument(f'--proxy-server={proxy}')
driver = webdriver.Chrome(service=service, options=chrome_options)
# Open the website
driver.get('https://www.google.com/')
wait = WebDriverWait(driver, 5)
search_box = wait.until(EC.presence_of_element_located((By.NAME, "q")))
proxy_success = True
break # exit the proxy lop
except Exception as e:
logging.error(f"Failed to open google for Address: {address} with proxy {proxy}- {str(e)}")
continue
if not proxy_success:
logging.error(f"All proxies failed for Address: {address}")
if driver is not None:
driver.close()
driver.quit()
continue #skip to the next address
found_results = False
results = []
try:
search_query = f"gasbuddy {address}"
logging.info(f'Searching for: {search_query} using proxy {proxy}')
# search address in Google
#search_box = driver.find_element(By.NAME, "q")
search_box.send_keys(search_query)
search_box.send_keys(Keys.RETURN)
# Use WebDriverWait instead of time.sleep
wait = WebDriverWait(driver, 8)
#wait.until(EC.presence_of_element_located((By.XPATH, "//a[h3]")))
wait.until(lambda driver: driver.execute_script('return document.readyState') == 'complete')
#wait.until(driver.find_elements(By.XPATH, "//a[h3]"))
#logging.info(f'Found search links for: {search_query} using proxy {proxy}')
time.sleep(5) # Wait for search results to load
# Find GasBuddy link in search results
links = driver.find_elements(By.XPATH, "//a[h3]")