多进程实现并发请求

Posted 2022-08-23 jintian

协程：在一个线程内指定切换

协程+异步IO

aiohttp　　　　　　asyncio　　模块

封装http数据包　　 异步IO

"""
可以实现并发
但是，请求发送出去后和返回之前，中间时期进程空闲
编写方式：
    - 直接返回处理
    - 通过回调函数处理
"""

########### 编写方式一 ###########
"""
from concurrent.futures import ProcessPoolExecutor
import requests
import time

def task(url):
    response = requests.get(url)
    print(url,response)
    # 写正则表达式


pool = ProcessPoolExecutor(7)
url_list = [
    ‘http://www.cnblogs.com/wupeiqi‘,
    ‘http://huaban.com/favorite/beauty/‘,
    ‘http://www.bing.com‘,
    ‘http://www.zhihu.com‘,
    ‘http://www.sina.com‘,
    ‘http://www.baidu.com‘,
    ‘http://www.autohome.com.cn‘,
]
for url in url_list:
    pool.submit(task,url)

pool.shutdown(wait=True)
"""

########### 编写方式二 ###########
from concurrent.futures import ProcessPoolExecutor
import requests
import time

def task(url):
    response = requests.get(url)
    return response

def done(future,*args,**kwargs):
    response = future.result()
    print(response.status_code,response.content)

pool = ProcessPoolExecutor(7)
url_list = [
    ‘http://www.cnblogs.com/wupeiqi‘,
    ‘http://huaban.com/favorite/beauty/‘,
    ‘http://www.bing.com‘,
    ‘http://www.zhihu.com‘,
    ‘http://www.sina.com‘,
    ‘http://www.baidu.com‘,
    ‘http://www.autohome.com.cn‘,
]
for url in url_list:
    v = pool.submit(task,url)
    v.add_done_callback(done)

pool.shutdown(wait=True)

多进程.py

import asyncio

"""
@asyncio.coroutine
def task():
    print(‘before...task......‘)
    yield from asyncio.sleep(5) # 发送Http请求，支持TCP获取结果..
    print(‘end...task......‘)


tasks = [task(), task()]

loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
"""


"""
import asyncio


@asyncio.coroutine
def task(host, url=‘/‘):
    print(‘start‘,host,url)
    reader, writer = yield from asyncio.open_connection(host, 80)

    request_header_content = "GET %s HTTP/1.0\\r\\nHost: %s\\r\\n\\r\\n" % (url, host,)
    request_header_content = bytes(request_header_content, encoding=‘utf-8‘)

    writer.write(request_header_content)
    yield from writer.drain()
    text = yield from reader.read()
    print(‘end‘,host, url, text)
    writer.close()

tasks = [
    task(‘www.cnblogs.com‘, ‘/wupeiqi/‘),
    task(‘dig.chouti.com‘, ‘/pic/show?nid=4073644713430508&lid=10273091‘)
]

loop = asyncio.get_event_loop()
results = loop.run_until_complete(asyncio.gather(*tasks))
loop.close()
"""

"""
import aiohttp
import asyncio


@asyncio.coroutine
def fetch_async(url):
    print(url)
    response = yield from aiohttp.request(‘GET‘, url)
    print(url, response)
    response.close()


tasks = [fetch_async(‘http://www.baidu.com/‘), fetch_async(‘http://www.chouti.com/‘)]

event_loop = asyncio.get_event_loop()
results = event_loop.run_until_complete(asyncio.gather(*tasks))
event_loop.close()
"""

import asyncio
import requests


@asyncio.coroutine
def task(func, *args):
    print(func,args)
    loop = asyncio.get_event_loop()
    future = loop.run_in_executor(None, func, *args) # requests.get(‘http://www.cnblogs.com/wupeiqi/‘)
    response = yield from future
    print(response.url, response.content)


tasks = [
    task(requests.get, ‘http://www.cnblogs.com/wupeiqi/‘),
    task(requests.get, ‘http://dig.chouti.com/pic/show?nid=4073644713430508&lid=10273091‘)
]

loop = asyncio.get_event_loop()
results = loop.run_until_complete(asyncio.gather(*tasks))
loop.close()

asyncio.py