爬虫自定义扩展-信号

Posted 2020-10-28 老王的农场

利用信号在指定的位置注册指定操作

 

from scrapy import signals


class MyExtension(object):
    def __init__(self, value):
        self.value = value

    @classmethod
    def from_crawler(cls, crawler):
        val = crawler.settings.getint(‘MMMM‘)
        ext = cls(val)

        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)

        return ext

    def spider_opened(self, spider):
        print(‘open‘)

    def spider_closed(self, spider):
        print(‘close‘)

View Code

 

#在scrapy中注册信号，spider_opend

crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)

#在scrapy中注册信号，spider_closed
crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)

 

#出发信号时，执行的函数

ext.spider_opened,  ext.spider_closed

def opend(self,spider):
    print(‘open‘)

def closed(self,spider):
    print(‘close‘)

 

setting中注册类的路径

# EXTENSIONS = {
#     # ‘step8_king.extensions.MyExtension‘: 500,
# }