scrapy pip中间件个人处理

Posted 2022-02-14 zengxm

import pymongo
from scrapy.conf import settings
import pymysql



class Mongodb_Pipeline(object):

    def __init__(self):
        self.client = pymongo.MongoClient()
        self.db = self.client[settings.get(‘MONGODB_NAME‘)]

    def process_item(self, item, spider):
        if item:
            coll = item[‘classification1‘]
            self.db[coll].insert(dict(item))
        return item


    def close_spider(self):
        self.client.close()

不成体系,不要看