多进程提速

Posted 2020-12-20 demo-deng

方式一：multiprocessing.Process

def worker(procnum, return_dict):
    ‘‘‘worker function‘‘‘
    print str(procnum) + ‘ represent!‘
    return_dict[procnum] = procnum

if __name__ == ‘__main__‘:
    manager = Manager()
    return_dict = manager.dict()
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,return_dict))
        jobs.append(p)
        p.start()

    for proc in jobs:
        proc.join()
    # 最后的结果是多个进程返回值的集合
    print return_dict.values()

方式二：Process

    pool = Pool(4)
    manager = Manager()
    return_dict = manager.dict()
    jobs = []
    q = Queue()

    with open(data_path, ‘r‘, encoding=‘utf8‘) as fr:
        content = fr.readlines()
        for d in content[:100]:
            q.put(d)
        print(‘数据放入队列完毕‘)

    for i in range(5):
        p = Process(target=func, args=(q, return_dict))
        jobs.append(p)
        p.start()
    for proc in jobs:
        proc.join()
    print(return_dict.values())

 方式三：Pool

import multiprocessing
import time


def func(msg):
    print(‘hello :‘, msg, time.ctime())
    time.sleep(2)
    print(‘end‘, time.ctime())
    return ‘done‘ + msg


if __name__ == ‘__main__‘:
    pool = multiprocessing.Pool(2)
    result = []
    for i in range(3):
        msg = ‘hello %s‘ % i
        result.append(pool.apply_async(func=func, args=(msg,)))

    pool.close()
    pool.join()

    for res in result:
        print(‘***:‘, res.get())             # get()函数得出每个返回结果的值

    print(‘All end--‘)