Scrapy 下载图片

Posted 2020-10-30 LightSong@计海拾贝

参考 ： https://www.jianshu.com/p/6c8d2730d088

https://docs.scrapy.org/en/latest/topics/item-pipeline.html#writing-your-own-item-pipeline

import scrapy

import requests
import os

class MeinvSpider(scrapy.Spider):
     name = "get_meinv"

    start_urls = [
         ‘https://www.du114.com/‘,
     ]

    def parse(self, response):

        dir_path = ‘%s/%s‘ % (".", self.name)

        if not os.path.exists(dir_path):
             os.makedirs(dir_path)

        for imggroup in response.css(‘div.Column-picBox‘):
             imgset = imggroup.css(‘ul>li img::attr("src")‘)
             for image_url in imgset.extract():

                print("image_url=%s" % image_url)

                us = image_url.split(‘/‘)[3:]
                 image_file_name = ‘_‘.join(us)
                 file_path = ‘%s/%s‘ % (dir_path, image_file_name)

                if os.path.exists(file_path):
                     continue

                with open(file_path, ‘wb‘) as handle:
                     response = requests.get(image_url, stream=True)
                     for block in response.iter_content(1024):
                         if not block:
                             break

                        handle.write(block)