scrapy+selenium 爬取淘宝

Posted hyxailj

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了scrapy+selenium 爬取淘宝相关的知识,希望对你有一定的参考价值。

# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from urllib.parse import quote
from ..items import ScrapyseleniumtestItem


class TaobaoSpider(scrapy.Spider):


    name = tao_bao
    allowed_domains = [www.taobao.com]
    base_url = https://s.taobao.com/search?q=

    def start_requests(self):
        # 通过self.settings.get()的方式获取setting里面的参数
        for keyword in self.settings.get(KEYWORDS):
            for page in range(1, self.settings.get(MAX_PAGE) + 1):
                url = self.base_url + quote(keyword)
                # 通过meta={‘键‘:"值"},传递你接下来需要的数据
                yield Request(url=url,
                              callback=self.parse,
                              meta={page:page},
                              dont_filter=True)

    def parse(self, response):
        # xpath中class中有空格(class="J_ItemPic img") 可以使用contains(@class,‘img‘)
        products = response.xpath(
            //div[@id="mainsrp-itemlist"]//div[@class="items"]//div[contains(@class, "item")])
        for product in products:
            item = ScrapyseleniumtestItem()
            item[price] = ‘‘.join(product.xpath(
                .//div[contains(@class, "price")]//text()).extract()).strip()
            item[title] = ‘‘.join(product.xpath(
                .//div[contains(@class, "title")]//text()).extract()).strip()
            item[shop] = ‘‘.join(product.xpath(
                .//div[contains(@class, "shop")]//text()).extract()).strip()
            item[image] = ‘‘.join(product.xpath(
                .//div[@class="pic"]//img[contains(@class, "img")]/@data-src).extract()).strip()
            item[deal] = product.xpath(
                .//div[contains(@class, "deal-cnt")]//text()).extract_first()
            item[location] = product.xpath(
                .//div[contains(@class, "location")]//text()).extract_first()
            yield item

# -*- coding: utf-8 -*-

# Scrapy settings for scrapyseleniumtest project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = scrapyseleniumtest

SPIDER_MODULES = [scrapyseleniumtest.spiders]
NEWSPIDER_MODULE = scrapyseleniumtest.spiders

MONGO_URI = localhost
MONGO_DB = taobao2

KEYWORDS = [iPad]
MAX_PAGE = 2
SELENIUM_TIMEOUT = 20
PHANTOMJS_SERVICE_ARGS = [--load-images=false, --disk-cache=true]

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = ‘scrapyseleniumtest (+http://www.yourdomain.com)‘

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘,
#   ‘Accept-Language‘: ‘en‘,
#}

# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    ‘scrapyseleniumtest.middlewares.ScrapyseleniumtestSpiderMiddleware‘: 543,
#}

# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   scrapyseleniumtest.middlewares.SeleniumMiddleware: 543,
}

# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    ‘scrapy.extensions.telnet.TelnetConsole‘: None,
#}

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   scrapyseleniumtest.pipelines.MongoPipeline: 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = ‘httpcache‘
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = ‘scrapy.extensions.httpcache.FilesystemCacheStorage‘


# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals


class ScrapyseleniumtestSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info(Spider opened: %s % spider.name)


from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.http import HtmlResponse
from logging import getLogger


class SeleniumMiddleware():
    def __init__(self, timeout=None, service_args=[]):
        self.logger = getLogger(__name__)
        self.timeout = timeout
        self.browser = webdriver.PhantomJS(service_args=service_args)
        self.browser.set_window_size(1400, 700)
        self.browser.set_page_load_timeout(self.timeout)
        self.wait = WebDriverWait(self.browser, self.timeout)

    def __del__(self):
        self.browser.close()

    def process_request(self, request, spider):
        """
        用PhantomJS抓取页面
        :param request: Request对象
        :param spider: Spider对象
        :return: HtmlResponse
        """
        self.logger.debug(PhantomJS is Starting)
        # 得到的是个int型的整数
        page = request.meta.get(page, 1)
        print(-----------------,page,-----------------------------)
        print(------------------,type(page),-----------------------------)
        try:
            self.browser.get(request.url)
            print(------------------, request.url,
                  -----------------------------)
            if page > 1:
                # 从第二页开始就要等待页面加载了
                # Presence_of_all_elements_located 判断一组元素是否存在
                input = self.wait.until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, #mainsrp-pager div.form > input)))
                # Element_to_be_clickable 判断元素是否点击,它处于可见和启动状态
                submit = self.wait.until(
                    EC.element_to_be_clickable((By.CSS_SELECTOR, #mainsrp-pager div.form > span.btn.J_Submit)))
                input.clear()
                # 输入第几页
                input.send_keys(page)
                submit.click()
            # Text_to_be_present_in_element 判断元素是否有xx文本信息
            self.wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, #mainsrp-pager li.item.active > span), str(page)))
            # Presence_of_all_elements_located 判断一组元素是否存在
            # 检测每一个item是否加载出来了
            self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, .m-itemlist .items .item)))
            return HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding=utf-8, status=200)
        except TimeoutException:
            return HtmlResponse(url=request.url, status=500, request=request)

    # 类方法 感觉就是获取setting里面的数据 在这里调用
    # 得到数据之后变成类变量
    @classmethod
    def from_crawler(cls, crawler):
        return cls(timeout=crawler.settings.get(SELENIUM_TIMEOUT),
                   service_args=crawler.settings.get(PHANTOMJS_SERVICE_ARGS))



 

 
# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy import Field


class ScrapyseleniumtestItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    collection = products
    image = Field()
    price = Field()
    deal = Field()
    title = Field()
    shop = Field()
    location = Field()

 

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


class ScrapyseleniumtestPipeline(object):
    def process_item(self, item, spider):
        return item

import pymongo


class MongoPipeline(object):

    #collection_name = ‘scrapy_items‘

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get(MONGO_URI),
            mongo_db=crawler.settings.get(MONGO_DB)
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        #
        # item.__class__.__name__  输出的item的类名
        name = item.__class__.__name__
        print(---------------name, name, -------------------)
        self.db[name].insert(dict(item))
        #self.db[item.collection].insert(dict(item))
        return item

 


 

 

以上是关于scrapy+selenium 爬取淘宝的主要内容,如果未能解决你的问题,请参考以下文章

scrapy+selenium 爬取淘宝商城商品数据存入到mongo中

Scrapy实战---Scrapy对接selenium爬取京东商城商品数据

使用scrapy爬虫,爬取今日头条首页推荐新闻(scrapy+selenium+PhantomJS)

scrapy按顺序启动多个爬虫代码片段(python3)

Scrapy+selenium爬取简书全站

Scrapy+selenium爬取简书全站