python3 进程池回调函数

Posted 2021-12-28 lilyxiaoyy

# coding:utf-8
import os
from multiprocessing import Pool


def func1(n):
    print("func1[%s]" % os.getpid())
    return n*n

def func2(m):
    print("func2[%s] 接收到的参数是:" % os.getpid(), m)


if __name__ == ‘__main__‘:
    print("主进程[%s]" % os.getpid())
    pool = Pool(5)
    for i in range(8):
        pool.apply_async(func1, args=(i,), callback=func2)
    pool.close()
    pool.join()

# func1[9136]
# func1[9136]
# func2[9672] 接收到的参数是: 0
# func2[9672] 接收到的参数是: 1
# func1[9136]
# func2[9672] 接收到的参数是: 4
# func1[9136]
# func2[9672] 接收到的参数是: 9
# func1[9136]
# func2[9672] 接收到的参数是: 16
# func1[9136]
# func2[9672] 接收到的参数是: 25
# func1[9136]
# func2[9672] 接收到的参数是: 36
# func1[9136]
# func2[9672] 接收到的参数是: 49


#回调函数在写的时候注意一点，回调函数的形参执行有一个，如果你的执行函数有多个返回值，那么也可以被回调
# 函数的这一个形参接收，接收的是一个元祖，包含着你执行函数的所有返回值。

 

使用多个进程来请求多个url来减少网络等待时间

# coding:utf-8
import os
import json
import requests
from multiprocessing import Pool


def get_url_content(url):
    print("[%s] %s" % (os.getpid(), url))
    response = requests.get(url)
    if response.status_code == 200:
        return ‘url‘: response.content.decode("utf-8")


def write_file(dic):
    with open("request.json", "a", encoding="utf-8") as f:
        json.dump(dic, f, ensure_ascii=False)
        f.write("\n")


if __name__ == ‘__main__‘:
    url_lst = [
        ‘https://www.baidu.com‘,
        ‘https://www.python.org‘,
        ‘https://www.openstack.org‘,
        ‘https://help.github.com/‘,
        ‘http://www.sina.com.cn/‘
    ]
    pool = Pool(5)
    for url in url_lst:
        pool.apply_async(get_url_content, args=(url,), callback=write_file)
    pool.close()
    pool.join()

 

 

无需回调函数实例

# coding:utf-8
import time
from multiprocessing import Pool


def work(n):
    time.sleep(1)
    return n**2


if __name__ == ‘__main__‘:
    p = Pool()
    res_l = []
    for i in range(10):
        res = p.apply_async(work, args=(i,))
        res_l.append(res)

    p.close()
    p.join()  # 等待进程池中所有进程执行完毕
    print([re.get() for re in res_l])  # 主进程拿到所有的处理结果,可以在主进程中进行统一进行处理


# [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]