scrapy爬取西刺网站ip

Posted 2020-09-26 道高一尺

# scrapy爬取西刺网站ip
# -*- coding: utf-8 -*-
import scrapy

from xici.items import XiciItem


class XicispiderSpider(scrapy.Spider):
    name = "xicispider"
    allowed_domains = ["www.xicidaili.com/nn"]
    start_urls = [‘http://www.xicidaili.com/nn/‘]

    def parse(self, response):
        item = XiciItem()
        for each in response.css(‘#ip_list tr‘):
            ip = each.css(‘td:nth-child(2)::text‘).extract_first()
            port = each.css(‘td:nth-child(3)::text‘).extract_first()
            if ip:
                ip_port = ip + ‘:‘ + port
                item[‘ip_port‘] = ip_port
                yield item

import pymongo

class XiciPipeline(object):

    collection_name = ‘scrapy_items‘

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
    #这里的from经常拼错啊
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get(‘MONGO_URI‘),
            mongo_db=crawler.settings.get(‘MONGO_DB‘)
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item))
        return item