Process Pool实现Python的并行执行

Posted 2020-10-04

参考：Python3.6.2文档

Source code: Lib/concurrent/futures/thread.py and Lib/concurrent/futures/process.py

 

Executor对象

class concurrent.futures.Executor

方法：

submit(fn, *args, **kwargs)                              #函数fn会按fn(*args **kwargs)执行，返回值是一个Future Object

map(func, *iterables, timeout=None, chunksize=1)                                 #executor.map() 函数调用时需要输入辅助函数和待处理的数据列表。

                                                                                     #这个函数帮我们完成所有麻烦的工作，把列表分成几个小列表，把小列表分配给每个子进程，运行子进程，以及汇总结果。

map(func, *iterables)  

submit(fn, *args, **kwargs)

shutdown(wait=True)

示例：

with ThreadPoolExecutor(max_workers=1) as executor:
    future = executor.submit(pow, 323, 1235)
    print(future.result())

 

ThreadPoolExecutor对象

Executor子类

class concurrent.futures.ThreadPoolExecutor(max_workers=None, thread_name_prefix=”)

import time
def wait_on_b():
    time.sleep(5)
    print(b.result())  # b will never complete because it is waiting on a.
    return 5

def wait_on_a():
    time.sleep(5)
    print(a.result())  # a will never complete because it is waiting on b.
    return 6


executor = ThreadPoolExecutor(max_workers=2)
a = executor.submit(wait_on_b)
b = executor.submit(wait_on_a)

 

ProcessPoolExecutor对象

Executor子类

class concurrent.futures.ProcessPoolExecutor(max_workers=None)

 

import concurrent.futures
import math

PRIMES = [
    112272535095293,
    112582705942171,
    112272535095293,
    115280095190773,
    115797848077099,
    1099726899285419]

def is_prime(n):
    if n % 2 == 0:
        return False

    sqrt_n = int(math.floor(math.sqrt(n)))
    for i in range(3, sqrt_n + 1, 2):
        if n % i == 0:
            return False
    return True

def main():
    with concurrent.futures.ProcessPoolExecutor() as executor:
        for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):
            print(‘%d is prime: %s‘ % (number, prime))

if __name__ == ‘__main__‘:
    main()

不并行的话，代码应该是

for number in PRIMES:
     pirme = is_prime(number);

ProcessPoolExecutor.map(...)常用于for循环中