Python多进程VS多线程

Posted 2020-10-28 fangbei

多线程：适用于处理I/O密集型任务和并发执行的阻塞操作
多进程：适用于处理计算密集型任务

 

# 多进程
import itertools
from concurrent.futures import ProcessPoolExecutor
result = []

# 回调函数，通过add_done_callback任务完成后调用
def when_done(r):
    # when_done在主进程中运行
    result.append(r.result())

"""
    with class_a() as a: 上下文管理器
"""
with ProcessPoolExecutor() as pool:
    for keep_stock_threshold, buy_change_threshold in             itertools.product(keep_stock_list, buy_change_list):

        """
            submit提交任务：使用calc函数和的参数通过submit提交到独立进程
            提交的任务必须是简单函数，进程并行不支持类方法、闭包等
            函数参数和返回值必须兼容pickle序列化，进程间的通信需要
        """
        future_result = pool.submit(calc, keep_stock_threshold,
                                    buy_change_threshold)
        # 当进程完成任务即calc运行结束后的回调函数
        future_result.add_done_callback(when_done)
print(sorted(result)[::-1][:10])

 

# 多线程
from concurrent.futures import ThreadPoolExecutor

result = []
def when_done(r):
    result.append(r.result())

with ThreadPoolExecutor(max_workers=8) as pool:
    for keep_stock_threshold, buy_change_threshold in             itertools.product(keep_stock_list, buy_change_list):
        future_result = pool.submit(calc, keep_stock_threshold,
                                    buy_change_threshold)
        future_result.add_done_callback(when_done)