python之multiprocessing

Posted RobotsRising

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了python之multiprocessing相关的知识,希望对你有一定的参考价值。

1、multiprocessing简介

  multiprocessing模块提供本地和远程并发性,通过使用子进程而不是线程来有效地避开全局解释器锁。由于这个原因,多处理模块允许程序员在给定的机器上充分利用多个处理器。它在Unix和Windows上运行。

2、进程的创建

2.1、创建一个进程

import multiprocessing,time


def run(name):
    time.sleep(2)
    print("hello", name)

if __name__ == "__main__":
    p = multiprocessing.Process(target=run, args=(bob,))
    p.start()
    p.join()

2.2、创建多个进程

import multiprocessing,time


def run(name):
    time.sleep(2)
    print("hello", name)

if __name__ == "__main__":
    for i in range(10):
        p = multiprocessing.Process(target=run, args=(bob %s % i,))
        p.start()
#         p.join()

2.3、进程中创建线程

import multiprocessing,time,threading

def thread_run():
    print("threading id :%s" %threading.get_ident())

def run(name):
    time.sleep(2)
    print("hello", name)
    t = threading.Thread(target=thread_run,)
    t.start()

if __name__ == "__main__":
    for i in range(10):
        p = multiprocessing.Process(target=run, args=(bob %s % i,))
        p.start()

2.4、获取进程id

from multiprocessing import Process
import os


def info(title):
    print(title)
    print(module name:, __name__)
    print(parent process:, os.getppid())
    print(process id:, os.getpid())
    print("

")


def f(name):
    info(33[31;1mcalled from child process function f33[0m)
    print(hello, name)

if __name__ == __main__:
    info(33[32;1mmain process line33[0m)

  输出结果:

main process line
module name: __main__
parent process: 6024
process id: 30756

2.5、获取进程id和子进程id

from multiprocessing import Process
import os


def info(title):
    print(title)
    print(module name:, __name__)
    print(parent process:, os.getppid())
    print(process id:, os.getpid())
    print("

")


def f(name):
    info(33[31;1mcalled from child process function f33[0m)
    print(hello, name)

if __name__ == __main__:
    info(33[32;1mmain process line33[0m)
    p = Process(target=f, args=(bob,))
    p.start()
    # p.join()

  输出结果:

main process line
module name: __main__
parent process: 6024
process id: 30756

3、进程间通信

  不同进程间内存是不共享的,要实现两个进程间的数据交换,可以使用一下方法:Queue,Pipes,Manager

3.1、Queue

from multiprocessing import Process, Queue

def f(q):
    q.put([42, None, hello])

if __name__ == __main__:
    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    
    print(q.get())

3.2、Pipes

from multiprocessing import Process, Pipe


def f(conn):
    conn.send([42, None, hello from child])
    conn.send([42, None, hello from child3])
    print("",conn.recv())
    conn.close()


if __name__ == __main__:
    parent_conn, child_conn = Pipe()
    p = Process(target=f, args=(child_conn,))
    p.start()
    print("parent",parent_conn.recv())  # prints "[42, None, ‘hello‘]"
    print("parent",parent_conn.recv())  # prints "[42, None, ‘hello‘]"
    parent_conn.send(" from hshs")  # prints "[42, None, ‘hello‘]"
    p.join()

3.3、Manager

from multiprocessing import Process, Manager
import os
def f(d, l):
    d[os.getpid()] =os.getpid()
    l.append(os.getpid())
    print(l)

if __name__ == __main__:
    with Manager() as manager:
        d = manager.dict() #{} #生成一个字典,可在多个进程间共享和传递

        l = manager.list(range(5))#生成一个列表,可在多个进程间共享和传递
        p_list = []
        for i in range(10):
            p = Process(target=f, args=(d, l))
            p.start()
            p_list.append(p)
        for res in p_list: #等待结果
            res.join()

        print(d)
        print(l)

3.4、进程同步

from multiprocessing import Process, Lock


def f(l, i):
    l.acquire()
    print(hello world, i)
    l.release()


if __name__ == __main__:
    lock = Lock()

    for num in range(100):
        Process(target=f, args=(lock, num)).start()

4、进程池

4.1、串行:apply

from  multiprocessing import Process, Pool,freeze_support
import time
import os

def Foo(i):
    time.sleep(2)
    print("in process",os.getpid())
    return i + 100

def Bar(arg):
    print(-->exec done:, arg,os.getpid())

if __name__ == __main__:
    #freeze_support()
    pool = Pool(processes=3) #允许进程池同时放入5个进程
    print("主进程",os.getpid())
    for i in range(10):
        pool.apply(func=Foo, args=(i,)) #串行
    print(end)
    pool.close()
    pool.join() #进程池中进程执行完毕后再关闭,如果注释,那么程序直接关闭。.join()

4.2、并行:apply_async

from  multiprocessing import Process, Pool,freeze_support
import time
import os

def Foo(i):
    time.sleep(2)
    print("in process",os.getpid())
    return i + 100

def Bar(arg):
    print(-->exec done:, arg,os.getpid())

if __name__ == __main__:
    #freeze_support()
    pool = Pool(processes=3) #允许进程池同时放入5个进程
    print("主进程",os.getpid())
    for i in range(10):
        pool.apply_async(func=Foo, args=(i,), callback=Bar) #callback=回调
        #pool.apply_async(func=Foo, args=(i,)) #串行
    print(end)
    pool.close()
    pool.join() #进程池中进程执行完毕后再关闭,如果注释,那么程序直接关闭。.join()

 

以上是关于python之multiprocessing的主要内容,如果未能解决你的问题,请参考以下文章

python并发之multiprocessing

Python系列之 - multiprocessing

python之multiprocessing

Python 多进程编程之multiprocessing--Pool

python 3 编程之多进程 multiprocessing模块

Python 多进程编程之multiprocessing--Process