scrapy爬虫案例

Posted 2021-03-04 hbfengjun

一个简单的爬虫案例

from scrapy_redis.spiders import RedisSpider
import os,urllib.request,time

class XiaohuaSpider(scrapy.Spider):
    name = ‘xiaohua‘
    allowed_domains = [‘90xiaohua.com‘]
    start_urls = [‘http://90xiaohua.com//‘]
    file_path = r‘D:python_codespideritemitemimg‘ # 图片保存位置

    def parse(self, response):
        time.sleep(1)
        li_list = response.xpath(‘//ul[@class="f-cb"]/li‘)
        print(response.request.url, response.meta.get(‘depth‘, 0))
        for item in li_list:
            # 获取img 信息
            data=item.xpath(‘.//a/img/@data-original‘).extract_first()
            if not data:
                continue

            src=‘http://90xiaohua.com%s‘%(data)
            src_img = data.rsplit(‘/‘,1)[1]  # 获取图片名称

            img = os.path.join(self.file_path, src_img)
            urllib.request.urlretrieve(src, img)   # 下载图片，保存到指定位置

        #分页
        page_list = response.xpath(‘//div[@class="m-page m-page-sr m-page-sm"]//a/@href‘).extract()
        for num in page_list:
            page = ‘http://90xiaohua.com/‘ + num
            yield Request(url=page, callback=self.parse, dont_filter=False)
            print(page)

案例