scrapy+selenium 爬取淘宝
Posted hyxailj
tags:
篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了scrapy+selenium 爬取淘宝相关的知识,希望对你有一定的参考价值。
# -*- coding: utf-8 -*- import scrapy from scrapy import Request from urllib.parse import quote from ..items import ScrapyseleniumtestItem class TaobaoSpider(scrapy.Spider): name = ‘tao_bao‘ allowed_domains = [‘www.taobao.com‘] base_url = ‘https://s.taobao.com/search?q=‘ def start_requests(self): # 通过self.settings.get()的方式获取setting里面的参数 for keyword in self.settings.get(‘KEYWORDS‘): for page in range(1, self.settings.get(‘MAX_PAGE‘) + 1): url = self.base_url + quote(keyword) # 通过meta={‘键‘:"值"},传递你接下来需要的数据 yield Request(url=url, callback=self.parse, meta={‘page‘:page}, dont_filter=True) def parse(self, response): # xpath中class中有空格(class="J_ItemPic img") 可以使用contains(@class,‘img‘) products = response.xpath( ‘//div[@id="mainsrp-itemlist"]//div[@class="items"]//div[contains(@class, "item")]‘) for product in products: item = ScrapyseleniumtestItem() item[‘price‘] = ‘‘.join(product.xpath( ‘.//div[contains(@class, "price")]//text()‘).extract()).strip() item[‘title‘] = ‘‘.join(product.xpath( ‘.//div[contains(@class, "title")]//text()‘).extract()).strip() item[‘shop‘] = ‘‘.join(product.xpath( ‘.//div[contains(@class, "shop")]//text()‘).extract()).strip() item[‘image‘] = ‘‘.join(product.xpath( ‘.//div[@class="pic"]//img[contains(@class, "img")]/@data-src‘).extract()).strip() item[‘deal‘] = product.xpath( ‘.//div[contains(@class, "deal-cnt")]//text()‘).extract_first() item[‘location‘] = product.xpath( ‘.//div[contains(@class, "location")]//text()‘).extract_first() yield item
# -*- coding: utf-8 -*- # Scrapy settings for scrapyseleniumtest project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html BOT_NAME = ‘scrapyseleniumtest‘ SPIDER_MODULES = [‘scrapyseleniumtest.spiders‘] NEWSPIDER_MODULE = ‘scrapyseleniumtest.spiders‘ MONGO_URI = ‘localhost‘ MONGO_DB = ‘taobao2‘ KEYWORDS = [‘iPad‘] MAX_PAGE = 2 SELENIUM_TIMEOUT = 20 PHANTOMJS_SERVICE_ARGS = [‘--load-images=false‘, ‘--disk-cache=true‘] # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = ‘scrapyseleniumtest (+http://www.yourdomain.com)‘ # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘, # ‘Accept-Language‘: ‘en‘, #} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # ‘scrapyseleniumtest.middlewares.ScrapyseleniumtestSpiderMiddleware‘: 543, #} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { ‘scrapyseleniumtest.middlewares.SeleniumMiddleware‘: 543, } # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # ‘scrapy.extensions.telnet.TelnetConsole‘: None, #} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { ‘scrapyseleniumtest.pipelines.MongoPipeline‘: 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = ‘httpcache‘ #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = ‘scrapy.extensions.httpcache.FilesystemCacheStorage‘
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class ScrapyseleniumtestSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. for i in result: yield i def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self, spider): spider.logger.info(‘Spider opened: %s‘ % spider.name) from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from scrapy.http import HtmlResponse from logging import getLogger class SeleniumMiddleware(): def __init__(self, timeout=None, service_args=[]): self.logger = getLogger(__name__) self.timeout = timeout self.browser = webdriver.PhantomJS(service_args=service_args) self.browser.set_window_size(1400, 700) self.browser.set_page_load_timeout(self.timeout) self.wait = WebDriverWait(self.browser, self.timeout) def __del__(self): self.browser.close() def process_request(self, request, spider): """ 用PhantomJS抓取页面 :param request: Request对象 :param spider: Spider对象 :return: HtmlResponse """ self.logger.debug(‘PhantomJS is Starting‘) # 得到的是个int型的整数 page = request.meta.get(‘page‘, 1) print(‘-----------------‘,page,‘-----------------------------‘) print(‘------------------‘,type(page),‘-----------------------------‘) try: self.browser.get(request.url) print(‘------------------‘, request.url, ‘-----------------------------‘) if page > 1: # 从第二页开始就要等待页面加载了 # Presence_of_all_elements_located 判断一组元素是否存在 input = self.wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, ‘#mainsrp-pager div.form > input‘))) # Element_to_be_clickable 判断元素是否点击,它处于可见和启动状态 submit = self.wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, ‘#mainsrp-pager div.form > span.btn.J_Submit‘))) input.clear() # 输入第几页 input.send_keys(page) submit.click() # Text_to_be_present_in_element 判断元素是否有xx文本信息 self.wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, ‘#mainsrp-pager li.item.active > span‘), str(page))) # Presence_of_all_elements_located 判断一组元素是否存在 # 检测每一个item是否加载出来了 self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ‘.m-itemlist .items .item‘))) return HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding=‘utf-8‘, status=200) except TimeoutException: return HtmlResponse(url=request.url, status=500, request=request) # 类方法 感觉就是获取setting里面的数据 在这里调用 # 得到数据之后变成类变量 @classmethod def from_crawler(cls, crawler): return cls(timeout=crawler.settings.get(‘SELENIUM_TIMEOUT‘), service_args=crawler.settings.get(‘PHANTOMJS_SERVICE_ARGS‘))
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy from scrapy import Field class ScrapyseleniumtestItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() collection = ‘products‘ image = Field() price = Field() deal = Field() title = Field() shop = Field() location = Field()
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don‘t forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html class ScrapyseleniumtestPipeline(object): def process_item(self, item, spider): return item import pymongo class MongoPipeline(object): #collection_name = ‘scrapy_items‘ def __init__(self, mongo_uri, mongo_db): self.mongo_uri = mongo_uri self.mongo_db = mongo_db @classmethod def from_crawler(cls, crawler): return cls( mongo_uri=crawler.settings.get(‘MONGO_URI‘), mongo_db=crawler.settings.get(‘MONGO_DB‘) ) def open_spider(self, spider): self.client = pymongo.MongoClient(self.mongo_uri) self.db = self.client[self.mongo_db] def close_spider(self, spider): self.client.close() def process_item(self, item, spider): # # item.__class__.__name__ 输出的item的类名 name = item.__class__.__name__ print(‘---------------name‘, name, ‘-------------------‘) self.db[name].insert(dict(item)) #self.db[item.collection].insert(dict(item)) return item
以上是关于scrapy+selenium 爬取淘宝的主要内容,如果未能解决你的问题,请参考以下文章
scrapy+selenium 爬取淘宝商城商品数据存入到mongo中
Scrapy实战---Scrapy对接selenium爬取京东商城商品数据