I need to download >400 hydrological datasets. The files that I need look like this https://waterdata.usgs.gov/nwis/dv?cb_00060=on&format=rdb&site_no=03251500&referred_module=sw&period=&begin_date=1979-12-31&end_date=2020-02-29 and are supposed to come out as comma separated textfiles.
My codes works as long as the data that I need are integers. As soon as there is one decimal number of the relevant data (e.g. 9.4), the rest of this file is skipped and a new dataset for another site is downloaded. My code is the following:
def check_and_write(data_id, request):
with open(path, “w”) as text_file:
text_file.write(“meanflow,day,month,year,date\n”)
found_mean = False
for line in request.iter_lines():
line_elements = line.decode(“utf-8”).split("\t")
if not found_mean:
pm, found_mean = find_mean_index(line_elements)
continue
if line_elements[0] == “USGS” and line_elements[1] == data_id:
if not line_elements[pm]:
meanflow_imperial = “None”
meanflow_metric = “None”
# continue # If you rather want to skip the missing values
else:
meanflow_imperial = line_elements[pm] + “,”
meanflow_metric = convert_meanflow(line_elements[pm]) + “,”
date = line_elements[2]
date_elements = date.split("-")
year = date_elements[0] + “,”
month = date_elements[1] + “,”
day = date_elements[2] + “,”
text_file.write(meanflow_metric + day + month + year + date + “\n”)
def find_mean_index(line_elements):
if line_elements[0] != “#”:
found = False
pm = 0
for element in line_elements:
for sub_element in element.split("_"):
if sub_element == “00003”:
found = True
elif sub_element == “cd”:
found = False
if found:
return pm, True
pm += 1
return None, False
def convert_meanflow(mflow):
# 1 cube feet = 1 ft x 1 ft x 1 ft = 0.3048 m x 0.3048 m x 0.3048 m = 0.028316846592 cubic meters
mflow_imperial = int(mflow)
mflow_metric = mflow_imperial * 0.028316846592
return str(mflow_metric)
checklist =
with open(‘textfile’) as usgsdata:
for row in csv.reader(usgsdata, delimiter=" “):
data_id = row[0][:]
url = (r"https://waterdata.usgs.gov/nwis/dv?cb_00060=on&format=rdb&site_no={0}&referred_module=sw&period=&begin_date=1979-12-31&end_date=2020-02-29”.format(data_id))
path = os.path.join(“path”, “{0}.txt”.format(data_id))
with requests.get(url, stream=True) as r:
if r.status_code == 200:
try:
check_and_write(data_id, r)
except:
checklist.append(data_id)
Any help is appreciated!