Scrapy学习-8-ItemLoader

Posted 2020-11-06 陈乾

ItemLoader使用

作用

　　方便管理维护重用xpath或css规则

 

实例

itemloader+图片处理

# items.py

    import scrapy
    from scrapy.loader.processors import MapCompose, Join, TakeFirst
    from scrapy.loader import ItemLoader
    import datetime
    import re

    def date_convert(value):
        try:
            create_date = datetime.datetime.strptime(value, "%Y/%m/%d").date()
        except Exception as e:
            create_date = datetime.datetime.now().date()

        return create_date

    def get_nums(value):
        match_re = re.match(".*?(\d+).*", value)
        if match_re:
            nums = int(match_re.group(1))
        else:
            nums = 0

        return nums

    def return_value(value):
        return value

    def remove_comment_tags(value):
        #去掉tag中提取的评论
        if "评论" in value:
            return ""
        else:
            return value

    class ArticleItemLoader(ItemLoader):
        #自定义itemloader
        default_output_processor = TakeFirst()

    class ArticlespiderItem(scrapy.Item):
        # title字段值回在最后加上 -jobbole-abc
        title = scrapy.Field(
            input_processor=MapCompose(lambda x:x+"-jobbole", lambda y:y+"-abc")
        )
        create_date = scrapy.Field(
            input_processor=MapCompose(date_convert),
        )
        url = scrapy.Field()
        url_object_id = scrapy.Field()
        # 由于front_image_url需要存储一个列表，不能使用defauls方法获取，为了保持原来的列表，我们重写一个output_processor
        front_image_url = scrapy.Field(
            output_processor=MapCompose(return_value)
        )
        front_image_path = scrapy.Field()
        praise_nums = scrapy.Field(
            input_processor=MapCompose(get_nums)
        )
        comment_nums = scrapy.Field(
            input_processor=MapCompose(get_nums)
        )
        fav_nums = scrapy.Field(
            input_processor=MapCompose(get_nums)
        )
        tags = scrapy.Field(
            input_processor=MapCompose(remove_comment_tags),
            output_processor=Join(",")
        )
        content = scrapy.Field()

# pipelines.py

    from scrapy.pipelines.images import ImagesPipeline

    class ArticleImagePipeline(ImagesPipeline):
        
        def item_completed(self, results, item, info):
            
            if "front_image_url" in item:
            
                for res, value in results:

                    image_path = value[‘path‘]

                    item[‘front_image_path‘] = image_path

            # 处理完成路径需要将item返回，因为在settings中，配置的了优先级，该pipelines可以将items继续传递给下一个pipelines中

            return item 

# 使用自己的pipelines

    ITEM_PIPELINES = {
        ‘ArticleSpider.pipelines.ArticlespiderPipeline‘: 300,
        ‘ArticleSpider.pipelines.ArticleImagePipeline‘: 1,
    }