python-自定义异步非阻塞爬虫框架

Posted Super-Code

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了python-自定义异步非阻塞爬虫框架相关的知识,希望对你有一定的参考价值。

api

import socket
import select


class MySock:
    def __init__(self, sock, data):
        self.sock = sock
        self.data = data

    def __getattr__(self, item):
        return getattr(self.sock, item)


class YinBing:
    def __init__(self):
        self.r_list = []
        self.w_list = []

    def add(self, req_info):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setblocking(0)
        try:
            sock.connect((req_info.get("host"), req_info.get("port")))
        except BlockingIOError:
            pass
        s = MySock(sock, req_info)
        self.r_list.append(s)
        self.w_list.append(s)

    def run(self):
        while True:
            rl, wl, el = select.select(self.r_list, self.w_list, [], 0.5)
            for sock in wl:
                sock.send(("GET %s Http/1.1\\r\\nHost:%s\\r\\n\\r\\n" % (sock.data["path"], sock.data["host"])).encode("utf-8"))
            for sock in rl:
                response = sock.recv(1024)
                callback = sock.data.get("callback")
                if callback: callback(response)
                self.r_list.remove(sock)

            if not self.r_list:
                break

调用方法

from 自定义爬虫框架 import YinBing


def done1(response):
    print(\'处理一\', response)


def done2(response):
    print(response)


url_list = [
    {\'host\': \'www.baidu.com\', \'port\': 80, \'path\': \'/\', \'callback\': done1},
    {\'host\': \'www.cnblogs.com\', \'port\': 80, \'path\': \'/index.html\', \'callback\': done2},
    {\'host\': \'www.bing.com\', \'port\': 80, \'path\': \'/\', \'callback\': None},
]

if __name__ == \'__main__\':
    y = YinBing()
    for obj in url_list:
        y.add(obj)
    y.run()

以上是关于python-自定义异步非阻塞爬虫框架的主要内容,如果未能解决你的问题,请参考以下文章

200行自定义异步非阻塞Web框架

200行自定义异步非阻塞Web框架

200行自定义异步非阻塞Web框架

200行自定义异步非阻塞Web框架

200行自定义异步非阻塞Web框架

Tornado----自定义异步非阻塞Web框架:Snow