scrapy+selenium 爬取淘宝商城商品数据存入到mongo中

Posted ray-mmss

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了scrapy+selenium 爬取淘宝商城商品数据存入到mongo中相关的知识,希望对你有一定的参考价值。

1.配置信息

# 设置mongo参数
MONGO_URI = localhost
MONGO_DB = taobao

# 设置搜索关键字
KEYWORDS=[小米手机,华为手机]
# 最大爬取页数
MAX_PAGE = 2
# 相应超时设置
SELENIUM_TIMEOUT = 20

ROBOTSTXT_OBEY = False #忽略

# 中间件
DOWNLOADER_MIDDLEWARES = {
   taobaoSpider.middlewares.SeleniumMiddleware: 300,
}


#项目管道
ITEM_PIPELINES = {
   # ‘taobaoSpider.pipelines.TaobaospiderPipeline‘: 300,
    taobaoSpider.pipelines.MongoPipeline: 400,
}
2.item
import scrapy


class TaobaospiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    price = scrapy.Field()
    sales = scrapy.Field()
    shop = scrapy.Field()
    location = scrapy.Field()
    image = scrapy.Field()

 3.spider

# -*- coding: utf-8 -*-
import scrapy

class TaobaoSpider(scrapy.Spider):
    name = taobao
    allowed_domains = [taobao.com]
    base_url = https://s.taobao.com/search?q=

    def start_url(self):
        # 通过self.settings.get()的方式获取setting里面的参数
        for keyword in self.gettings.get(KEYWORDS):
            for page in range(1,self.gettings.get(MAX_PAGE)+1):
                url = self.base_url.format(self.url)
                yield scrapy.Request(url=self.url,
                                     callback=self.parse,
                                    meta={page:page}, # 传递页码
                                    dont_filter=True) # 不去重


    def parse(self, response):

        products = response.xpath(//*[@id="mainsrp-itemlist"]/div[@class="m-itemlist"]/div[@class="grid g-claerfix"]/div[1])

        # products = response.xpath(‘//div[contains(@class,"item J_MouserOnverReq"/‘)
        for product in products:
            from taobaoSpider.taobaoSpider.items import TaobaospiderItem
            item = TaobaospiderItem()
            item[title] = ‘‘.join(product.xpath(//div[contains(@class,"title")]/text()).extract()).strip()
            item[location] = ‘‘.join(product.xpath(//div[contains(@class,"location")]/text()).extract()).strip()
            item[shop] = ‘‘.join(product.xpath(//div[contains(@class,"shop")]/text()).extract()).strip()
            item[price] = ‘‘.join(product.xpath(//div[contains(@class,"price")]/text()).extract()).strip()
            item[deal] = ‘‘.join(product.xpath(//div[contains(@class,"deal-cnt")]/text()).extract()).strip()
            item[iamge] = ‘‘.join(product.xpath(//div[@class="pic"]/img[contains(@class,"img")/@data-src).extract()).strip()
            yield item

4.中间件

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.http import htmlResponse
from logging import getLogger


class SeleniumMiddleware(object):
    # def __init__(self, timeout=None, service_args=[]):
    def __init__(self, timeout=None):
        self.logger = getLogger(__name__)
        self.timeout = timeout
        # self.browser = webdriver.PhantomJS(service_args=service_args)
        # 无界面模式
        # self.options = webdriver.ChromeOptions()
        # self.options.add_argument(‘--headless‘)
        # self.browser = webdriver.Chrome(chrome_options=self.options)
        self.browser = webdriver.Chrome()
        # self.browser.set_window_size(1400, 700)
        self.browser.set_page_load_timeout(self.timeout)
        self.wait = WebDriverWait(self.browser, self.timeout)
        print(timeout:, self.timeout)

    def __del__(self):
        self.browser.close()

    def process_request(self, request, spider):
        ‘‘‘
        :param request:
        :param spider:
        :return:
        ‘‘‘
        self.logger.debug(Selenium is Runing)
        # 得到的是个int型的整数
        page = request.meta.get(page, 1)
        try:
            self.browser.get(request.url)
            print(10*-, request.url,10*-)
            if page > 1:
                # 从第二页开始,等待页面加载完成
                # Presence_of_all_elements_located 判断一组元素是否存在
                input = self.wait.until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, ##mainsrp-pager > div > div > div > div.form > input)))
                # Element_to_be_clickable 判断元素是否可点击
                submit = self.wait.until(
                    EC.element_to_be_clickable((By.CSS_SELECTOR, #mainsrp-pager > div > div > div > div.form > span.btn.J_Submit)))
                input.clear()
                # 输入第几页
                input.send_keys(page)
                submit.click()
            # Text_to_be_present_in_element 判断元素是否有xx文本信息
            self.wait.until(EC.text_to_be_present_in_element((
                By.CSS_SELECTOR, #mainsrp-pager > div > div > div > ul > li.item.active > span), str(page)))
            # Presence_of_all_elements_located 判断一组元素是否存在
            # 检测每一个item是否加载出来了
            self.wait.until(EC.presence_of_element_located((
                By.CSS_SELECTOR, #mainsrp-itemlist .m-itemlist .grid.g-clearfix .item)))
            return HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding=utf-8, status=200)
        except TimeoutException:
            return HtmlResponse(url=request.url, status=500, request=request)

    # 类方法 感觉就是获取setting里面的数据 在这里调用
    # 得到数据之后变成类变量
    @classmethod
    def from_crawler(cls, crawler):
        return cls(timeout=crawler.settings.get(SELENIUM_TIMEOUT),)
                   # service_args=crawler.settings.get(‘PHANTOMJS_SERVICE_ARGS‘))

5.管道(存储到mongo中)

class Pipeline(object):
    def process_item(self, item, spider):
        return item

import pymongo

# 存储到mongo中
class MongoPipeline(object):

    # 配置mongo数据库
    def __init__(self,mongo_url,mongo_db):
        self.mongo_url = mongo_url
        self.mongo_db = mongo_db

    # 从setting中获取参数
    @classmethod
    def from_crawler(cls,crawler):
        mongo_url = crawler.settings.get(MONGO_URL)
        mongo_db = crawler.settings.get(MONGO_DB)

    # 连接数据库
    def open_spider(self,spider):
        self.client = pymongo.MongoClient(self.mongo_url)
        self.db = self.client[self.mongo_db]

    # 关闭数据库连接
    def close_spider(self,spider):
        self.client.close()

    # 设置存储格式
    def process_item(self,item,spider):
        # item.__class__.__name__  输出的item的类名
        name = item.__class__.__name__
        print(---------------name, name, -------------------)
        self.db[name].insert(dict(item))
        return item

 

以上是关于scrapy+selenium 爬取淘宝商城商品数据存入到mongo中的主要内容,如果未能解决你的问题,请参考以下文章

scrapy+selenium 爬取淘宝

scrapy结合selenium爬取淘宝等动态网站

如何爬取淘宝登录页面

用scrapy爬取京东商城的商品信息

Scrapy+selenium爬取简书全站

Scrapy+selenium爬取简书全站