Scrapy框架第一个爬虫项目--汽车之家二手车列表信息抓取

Posted 2021-01-01 tmd201304

废话不多说，上代码

1、以下代码为spider文件

import scrapy
from car_home.items import che168Item

class Che168Spider(scrapy.Spider):
    name = ‘che168‘
    allowed_domains = [‘che168.com‘]
    start_urls = [‘https://www.che168.com/beijing/list/‘]
    
    def parse(self, response):
        #获取多个列表页链接，爬取多页数据
        max_page = response.xpath(‘//div[@id="listpagination"]/a/text()‘)[-1].extract()
        
        base_url = ‘https://www.che168.com/beijing/a0_0msdgscncgpi1ltocsp{}exx0/‘
        for i in range(1,int(max_page)+1):
            url = base_url.format(i)
            yield scrapy.Request(url,callback=self.parse_list)
            
        
    def parse_list(self, response):
        titles = response.xpath(‘//div[@class="title"]/a/text()‘).extract()
        travel = response.xpath(‘//div[@class="details"]/ul/li[1]/span[1]/text()‘).extract()
        year = response.xpath(‘//div[@class="details"]/ul/li[2]/span[1]/text()‘).extract()
        type = response.xpath(‘//div[@class="details"]/ul/li[3]/span[1]/text()‘).extract()
        address = response.xpath(‘//div[@class="details"]/ul/li[4]/span[1]/text()‘).extract()
        standard = response.xpath(‘//div[@class="details"]/ul/li[5]/span[1]/text()‘).extract()
        nums = response.xpath(‘//div[@class="details"]/ul/li[6]/span[1]/text()‘).extract()
        item = che168Item()
        for i in range(0,len(titles)):
            item[‘titles‘] = titles[i]
            item[‘travel‘] = travel[i]
            item[‘year‘] = year[i]
            item[‘type‘] = type[i]
            item[‘address‘] = address[i]
            item[‘standard‘] = standard[i]
            item[‘nums‘] = nums[i]
            #print(item)
            
            yield item

2、items

class che168Item(scrapy.Item):
    titles =scrapy.Field()
    travel =scrapy.Field()
    year =scrapy.Field()
    type =scrapy.Field()
    address =scrapy.Field()
    standard =scrapy.Field()
    nums =scrapy.Field()
    
    def get_insert_sql(self):
        sql = ‘insert into car_home(titles,travel,year,type,address,standard,nums) values(%s, %s, %s, %s, %s, %s, %s)‘
        data = (self[‘titles‘],self[‘travel‘],self[‘year‘],self[‘type‘],self[‘address‘],self[‘standard‘],self[‘nums‘])
        return (sql,data)

3、pipeline

from car_home.sql_unit import SqlUnit

class CarHomePipeline(object):
    def process_item(self, item, spider):
        return item

class che168Pipeline(object):
    
    def process_item(self, item, spider):
        s = SqlUnit()
        (sql,data) = item.get_insert_sql()
        s.execute_sql(sql,data)
        return item

4、数据库存储封装 

import pymysql

class SqlUnit(object):
    
    def __init__(self):
        self.db = pymysql.connect("127.0.0.1","root","123456","home")
        self.cursor = self.db.cursor()
    
    def execute_sql(self,sql,data=None):
        self.cursor.execute(sql,data)
        self.db.commit()
    
    def __del__(self):
        self.cursor.close()
        self.db.close()

 5、settings中将che168Pipeline加入ITEM_PIPELINES