Hello, for a freelancing project I’m working on scraping data from HVAC websites, and this website here is giving me trouble.
What I want my code to do is take each zipcode in an array, enter it, submit, and scrape data like dealer names and addresses for each.
My problem is that I don’t get past the submit step. When webdriver is not run in headless mode, I see the zipcode entered. Yet the page gives a “Enter valid zip/postal code” error
Here is the relevant code:
```python
# Import required modules
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
import codecs
import re
from time import sleep
#importing zipcode array
from array import array
from zipcode_list import zip_arr
#importing csv
import csv
#time to wait for element to appear
timeout = 20
#checking that dealers have been found
def is_multiple_dealers(driver):
dealer = driver.find_elements(By.XPATH, "//div[contains(@class, 'dl-dealer-count__block')]/span")
for d in dealer:
if '0 dealers near' in d.text:
return False, d.text
break
else:
return True, d.text
break
break
def main():
ffOptions = webdriver.FirefoxOptions()
ffOptions.set_preference("geo.enabled", False)
ffOptions.set_preference('geo.prompt.testing', False)
ffOptions.set_preference('geo.prompt.testing.allow', False)
driver = webdriver.Firefox(options=ffOptions)
wait = WebDriverWait(driver, timeout)
a = zip_arr
zipcode_input = a[0]
#Redirect
val = 'https://www.goodmanmfg.com/support/find-a-dealer'
driver.get(val)
wait.until(EC.url_to_be(val))
page_source = driver.page_source
sleep(1)
driver.refresh()
#setting up arrays for csv
zipcode_status_array = []
names_array = []
address_array = []
phone_array = []
for x in range(len(a)):
zipcode_input = a[x]
#waiting until input box is there then typing zipcode
zipcode_form = wait.until(EC.element_to_be_clickable((By.ID, 'zipcode')))
zipcode_form.click()
zipcode_form.clear()
driver.execute_script("arguments[0].value = '" + str(zipcode_input) + "';", zipcode_form)
sleep(2)
#selecting 10 in the "Miles From Home" dropdown
select = Select(driver.find_element(By.ID, 'miles'))
select.select_by_value('10')
#waiting until submit button is there then submitting
sleep(2)
submit_zipcode = wait.until(EC.presence_of_element_located((By.XPATH, "//button[contains(., 'SEARCH')]")))
driver.execute_script("arguments[0].click();", submit_zipcode)
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div.dl-dealer-details')))
dealer_boolean, dealer_quantity_text = is_multiple_dealers(driver)
if dealer_boolean == True:
print(dealer_quantity_text)
print("")
dealer_items = driver.find_elements(By.CSS_SELECTOR, 'div.dl-dealer-details')
for dealer in dealer_items:
#code for finding elements on dealers page and appending to respective arrays
print("")
print("")
else:
#append "NAN" to relevant arrays for entry
print("")
print("")
sleep(1)
#code here for writing arrays to csv, not relevant
print("complete")
driver.quit()
if __name__ == "__main__":
main()
I first tried using sendkeys(), then currently javascript, but neither were recognized by the website.
My current traceback is a TimeoutException for
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div.dl-dealer-details')))
which again just supports that my zipcode inputting code is running, it’s just that the website isn’t accepting it and proceeding?
This is my first post here, but I’m just really hoping I can get someone more knowledgeable to finally solve this, since the alternative has been hours of entering data manually. Anything would be appreciated!