for some reason Scrapy stops rendering pages after page 4, why is this?
My code:
import scrapy
from scrapy_splash import SplashRequest
import csv
class Description(scrapy.Spider):
name = 'Description'
auction_URL = 'https://www.ebay.co.uk/sch/i.html?_from=R40&_nkw=toaster&_sacat=0&LH_Sold=1&LH_Complete=1&rt=nc&_pgn=1'
def start_requests(self):
yield SplashRequest(url=self.auction_URL, callback=self.parse,
endpoint='render.html',
args={'wait': 3},
)
def parse(self, response):
items = []
for product in response.css('li.s-item'):
title = product.css('h3.s-item__title.s-item__title--has-tags::text').get()
subtitle = product.css('div.s-item__subtitle::text').get()
productURL = product.css('a.s-item__link::attr(href)').get()
for x in product.css('div.s-item__details.clearfix'):
price = x.css('span.POSITIVE::text').get()
item = {
"title": title,
"subtitle": subtitle,
"price": price,
"url": productURL
}
items.append(item)
keys = items[0].keys()
with open(r'C:\Users\Hugo\Documents\CSV export\info.csv', 'a', newline='') as output_file:
dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writerows(items)
As you can see, it works on the page currently in place. However if you change it to:
'https://www.ebay.co.uk/sch/i.html?_from=R40&_nkw=toaster&_sacat=0&LH_Sold=1&LH_Complete=1&rt=nc&_pgn=6'
It no longer finds anything, been trying to work out why this is for a while now haha1.