sn图书spider

Posted 2021-01-22 xu12135

# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy

class SnbookSpider(scrapy.Spider):
    name = ‘snbook‘
    allowed_domains = [‘suning.com‘]
    start_urls = [‘https://book.suning.com/‘]

    def parse(self, response):
        div_list=response.xpath(‘//div[@class="menu-list"]/div[@class="menu-item"]‘)
        div_sub_list=response.xpath(‘//div[@class="menu-list"]/div[@class="menu-sub"]‘)
        for i in div_list:
            item={}
            # 大分类的名字
            item[‘b_name‘]=i.xpath(".//h3/a/text()").extract_first()
            # 当前大分类下的中间分类
            # a_list=i.xpath(‘.//dd/a‘).extract()
            #
            # for i in a_list:
            #     # 中分类的名字
            #     item[‘m_name‘] =i.xpath(‘.//a/text()‘).extract_first()

            for i in div_sub_list:
                # 中分类的名字
                item[‘m_name‘]=i.xpath(‘.//p[@class="submenu-item"]/a/text()‘).extract_first()

                li_list=i.xpath(‘.//ul[@class="book-name-list clearfix"]/li‘)
                for i in li_list:
                    # 小分类的名字
                    item[‘s_name‘]=i.xpath(‘./a/text()‘).extract_first()

                    # 小分类的URL地址
                    item["s_href"] = i.xpath("./a/@href").extract_first()

                    # 请求图书的列表页
                    yield scrapy.Request(
                        item["s_href"],
                        callback=self.parse_book_list,
                        meta={"item": deepcopy(item)}
                    )



    def parse_book_list(self,response):
        item = response.meta["item"]
        li_list=response.xpath(‘//ul[@class="clearfix"]/li‘)
        for i in li_list:
            # #书名
            item[‘book_name‘]=i.xpath(‘.//p[2]/a/text()‘).extract_first().strip()
            # 书的url地址，不完整
            item[‘book_href‘]=i.xpath(‘.//p[2]/a/@href‘).extract_first()
            detail_url=‘https:‘+item[‘book_href‘]

            # 书店名
            item[‘book_store_name‘]=i.xpath(‘.//p[4]/a/text()‘).extract_first()

            # 请求图书详情页
            yield scrapy.Request(
                detail_url,
                callback=self.detail_book_list,
                meta={"item": deepcopy(item)}
            )
        page_list=response.xpath(‘//div[@id="bottom_pager"]/a‘)

        for i in page_list:
            page=i.xpath(‘./@href‘).extract_first()
            page_url=‘https://list.suning.com‘+page
            yield scrapy.Request(
                url=page_url,
                callback=self.parse_book_list,
                meta={‘item‘:item}
            )

    def detail_book_list(self,response):
        item = response.meta["item"]
        import re
        item[‘book_price‘]=re.findall(‘"itemPrice":"(.*?)",‘,response.body.decode())[0]


        print(item)