加速scrapy

Posted 2020-10-17 不可叽叽歪歪

def parse(self, response):
# Get the next index URLs and yield Requests
next_sel = response.xpath(‘//*[contains(@class,"next")]//@href‘) for url in next_sel.extract():
yield Request(urlparse.urljoin(response.url, url))

# Iterate through products and create PropertiesItems selectors = response.xpath(
‘//*[@itemtype="http://schema.org/Product"]‘) for selector in selectors:
yield self.parse_item(selector, response)