爬虫相关

Posted hejunqing

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了爬虫相关相关的知识，希望对你有一定的参考价值。

性能相关的了解

在编写爬虫时，性能的消耗主要在IO请求中，当单进程单线程模式下请求URL时必然会引起等待，从而使得请求整体变慢。如下各种不同的执行

技术分享图片 
import requests

def fetch_async(url):
    response = requests.get(url)
    return response


url_list = [‘http://www.github.com‘, ‘http://www.bing.com‘]

for url in url_list:
    fetch_async(url)
同步执行
技术分享图片 
from concurrent.futures import ThreadPoolExecutor
import requests


def fetch_async(url):
    response = requests.get(url)
    return response


url_list = [‘http://www.github.com‘, ‘http://www.bing.com‘]
pool = ThreadPoolExecutor(5)
for url in url_list:
    pool.submit(fetch_async, url)
pool.shutdown(wait=True)
多线程
技术分享图片 
from concurrent.futures import ThreadPoolExecutor
import requests

def fetch_async(url):
    response = requests.get(url)
    return response


def callback(future):
    print(future.result())


url_list = [‘http://www.github.com‘, ‘http://www.bing.com‘]
pool = ThreadPoolExecutor(5)
for url in url_list:
    v = pool.submit(fetch_async, url)
    v.add_done_callback(callback)
pool.shutdown(wait=True)
多线程+回调函数
技术分享图片 
from concurrent.futures import ProcessPoolExecutor
import requests


def fetch_async(url):
    response = requests.get(url)
    return response


def callback(future):
    print(future.result())


url_list = [‘http://www.github.com‘, ‘http://www.bing.com‘]
pool = ProcessPoolExecutor(5)
for url in url_list:
    v = pool.submit(fetch_async, url)
    v.add_done_callback(callback)
pool.shutdown(wait=True)
多进程+回调函数

通过上述代码均可以完成对请求性能的提高，对于多线程和多进行的缺点是在IO阻塞时会造成了线程和进程的浪费，所以异步IO会是首选，通常的异步io模型如下：

技术分享图片 
import asyncio


@asyncio.coroutine
def func1():
    print(‘before...func1......‘)
    yield from asyncio.sleep(5)
    print(‘end...func1......‘)


tasks = [func1(), func1()]

loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
asyncio1  

 #只能基于发tcp请求

技术分享图片 
import asyncio


@asyncio.coroutine
def fetch_async(host, url=‘/‘):
    print(host, url)
    reader, writer = yield from asyncio.open_connection(host, 80)

    request_header_content = """GET %s HTTP/1.0\\r\\nHost: %s\\r\\n\\r\\n""" % (url, host,)
    request_header_content = bytes(request_header_content, encoding=‘utf-8‘)

    writer.write(request_header_content)
    yield from writer.drain()
    text = yield from reader.read()
    print(host, url, text)
    writer.close()

tasks = [
    fetch_async(‘www.cnblogs.com‘, ‘/wupeiqi/‘),
    fetch_async(‘dig.chouti.com‘, ‘/pic/show?nid=4073644713430508&lid=10273091‘)
]

loop = asyncio.get_event_loop()
results = loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
asyncio2
技术分享图片 
import aiohttp
import asyncio


@asyncio.coroutine
def fetch_async(url):
    print(url)
    response = yield from aiohttp.request(‘GET‘, url)
    # data = yield from response.read()
    # print(url, data)
    print(url, response)
    response.close()


tasks = [fetch_async(‘http://www.google.com/‘), fetch_async(‘http://www.chouti.com/‘)]

event_loop = asyncio.get_event_loop()
results = event_loop.run_until_complete(asyncio.gather(*tasks))
event_loop.close()
asyncio+aiohttp
技术分享图片 
import asyncio
import requests


@asyncio.coroutine
def fetch_async(func, *args):
    loop = asyncio.get_event_loop()
    future = loop.run_in_executor(None, func, *args)
    response = yield from future
    print(response.url, response.content)


tasks = [
    fetch_async(requests.get, ‘http://www.cnblogs.com/wupeiqi/‘),
    fetch_async(requests.get, ‘http://dig.chouti.com/pic/show?nid=4073644713430508&lid=10273091‘)
]

loop = asyncio.get_event_loop()
results = loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
asyncio+requests
技术分享图片 
import gevent

import requests
from gevent import monkey

monkey.patch_all()


def fetch_async(method, url, req_kwargs):
    print(method, url, req_kwargs)
    response = requests.request(method=method, url=url, **req_kwargs)
    print(response.url, response.content)

# ##### 发送请求 #####
gevent.joinall([
    gevent.spawn(fetch_async, method=‘get‘, url=‘https://www.python.org/‘, req_kwargs={}),
    gevent.spawn(fetch_async, method=‘get‘, url=‘https://www.yahoo.com/‘, req_kwargs={}),
    gevent.spawn(fetch_async, method=‘get‘, url=‘https://github.com/‘, req_kwargs={}),
])

# ##### 发送请求（协程池控制最大协程数量） #####
# from gevent.pool import Pool
# pool = Pool(None)
# gevent.joinall([
#     pool.spawn(fetch_async, method=‘get‘, url=‘https://www.python.org/‘, req_kwargs={}),
#     pool.spawn(fetch_async, method=‘get‘, url=‘https://www.yahoo.com/‘, req_kwargs={}),
#     pool.spawn(fetch_async, method=‘get‘, url=‘https://www.github.com/‘, req_kwargs={}),
# ])
gevent+requests
技术分享图片 
import grequests


request_list = [
    grequests.get(‘http://httpbin.org/delay/1‘, timeout=0.001),
    grequests.get(‘http://fakedomain/‘),
    grequests.get(‘http://httpbin.org/status/500‘)
]


# ##### 执行并获取响应列表 #####
# response_list = grequests.map(request_list)
# print(response_list)


# ##### 执行并获取响应列表（处理异常） #####
# def exception_handler(request, exception):
# print(request,exception)
#     print("Request failed")

# response_list = grequests.map(request_list, exception_handler=exception_handler)
# print(response_list)
grequests
技术分享图片 
from twisted.web.client import getPage, defer
from twisted.internet import reactor


def all_done(arg):
    reactor.stop()


def callback(contents):
    print(contents)


deferred_list = []

url_list = [‘http://www.bing.com‘, ‘http://www.baidu.com‘, ]
for url in url_list:
    deferred = getPage(bytes(url, encoding=‘utf8‘))
    deferred.addCallback(callback)
    deferred_list.append(deferred)

dlist = defer.DeferredList(deferred_list)
dlist.addBoth(all_done)

reactor.run()
twisted示例
技术分享图片 
from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import ioloop


def handle_response(response):
    """
    处理返回值内容（需要维护计数器，来停止IO循环），调用 ioloop.IOLoop.current().stop()
    :param response: 
    :return: 
    """
    if response.error:
        print("Error:", response.error)
    else:
        print(response.body)


def func():
    url_list = [
        ‘http://www.baidu.com‘,
        ‘http://www.bing.com‘,
    ]
    for url in url_list:
        print(url)
        http_client = AsyncHTTPClient()
        http_client.fetch(HTTPRequest(url), handle_response)


ioloop.IOLoop.current().add_callback(func)
ioloop.IOLoop.current().start()
tornado
技术分享图片 
from twisted.internet import reactor
from twisted.web.client import getPage
import urllib.parse


def one_done(arg):
    print(arg)
    reactor.stop()

post_data = urllib.parse.urlencode({‘check_data‘: ‘adf‘})
post_data = bytes(post_data, encoding=‘utf8‘)
headers = {b‘Content-Type‘: b‘application/x-www-form-urlencoded‘}
response = getPage(bytes(‘http://dig.chouti.com/login‘, encoding=‘utf8‘),
                   method=bytes(‘POST‘, encoding=‘utf8‘),
                   postdata=post_data,
                   cookies={},
                   headers=headers)
response.addBoth(one_done)

reactor.run()
twisted更多
技术分享图片 
import select
import socket

class WrapSpider(object):
    ‘‘‘包装sock对象‘‘‘
    def __init__(self,sock,web_info):
        self.sock=sock
        self.web_info=web_info
    def fileno(self):
        return self.sock.fileno()       #####容易忘记

class GreatIO(object):

    def __init__(self):
        self.coons=[]
        self.sock_list=[]

    def add_spider(self,web_info):
        ‘‘‘添加相关请求‘‘‘
        sock=socket.socket()
        sock.setblocking(False)
        try:
            sock.connect((web_info[‘host‘],web_info[‘port‘]))
        except BlockingIOError as e:
            pass
        obj=WrapSpider(sock,web_info)
        self.sock_list.append(obj)
        self.coons.append(obj)

    def run(self):
        """
        执行请求
        :return:
        """
        while True:
            #select.select([对象])
            #可以是任意对象，但是对象要有fileno方法
            #现在对其对象进行包装，obj 是WrapSpider对象
            r, w, e = select.select(self.sock_list,self.coons, [], 0.05)
            for obj in w:       #连接成功的
                data=‘GET %s HTTP/1.1\\r\\nhost:%s\\r\\n\\r\\n‘.format(obj.web_info[‘path‘],obj.web_info[‘host‘]).encode(‘utf-8‘)
                obj.sock.send(data)
                self.coons.remove(obj)
            for obj in r:
                response=obj.sock.recv(8096)
                obj.web_info[‘callback‘](obj.web_info[‘host‘],response)
                self.sock_list.remove(obj)
            if not self.sock_list:
                break







def done(host,callbac):
    print(host,callbac)

Spider_list=[
    {‘host‘:‘www.baidu.com‘, ‘port‘:80, ‘path‘:‘/index.php?tn=06074089_9_pg‘, ‘callback‘:done},
    {‘host‘:‘www.dig.chouti.com‘, ‘port‘: 80,‘path‘: ‘/‘,‘callback‘:done},
    {‘host‘:‘www.bing.com‘,‘port‘:80, ‘path‘:‘/‘, ‘callback‘:done},
]

qing=GreatIO()
for sp in Spider_list:
    qing.add_spider(sp)

qing.run()
以上的类似原理（通过select）

 

以上是关于爬虫相关的主要内容，如果未能解决你的问题，请参考以下文章

AJAX相关JS代码片段和部分浏览器模型

大数据时代，爬虫技术相关法律问题介绍

python爬虫相关基础概念

Jekyll 偏移代码片段高亮的初始行

从零开始配置vim(27)——代码片段

NIH周三讲座视频爬虫