Python多进程与单进程效率对比

Posted 2021-02-15 开心大人

运行环境：Python3 in win10
先生成200个测试文件

# generate.py
i = 0
while i < 200:
    o = open("test\" + str(i) + ".py", "w")
    content = str(i)
    o.write(content)
    o.close()
    i += 1

多进程拷贝文件

# multi-pool-copy.py
from multiprocessing import Pool, Manager
import time
import os
import shutil
import random

start = time.time()

def copyFile(file_name, old_folder_name, new_folder_name, q):
    time.sleep(random.random())
    shutil.copyfile(old_folder_name + '\' + file_name, new_folder_name + '\' + file_name,)
    q.put(file_name)  # put item into the queue

def main():
    pool = Pool(5)
    q = Manager().Queue()

    old_folder_name = input("Please input the folder name you want to copy: ")

    new_folder_name = old_folder_name + "-copy"
    os.mkdir(new_folder_name)

    file_name_list = os.listdir(old_folder_name)

    for file in file_name_list:
        pool.apply_async(copyFile, args=(file, old_folder_name, new_folder_name, q))
    
    cnt = 0
    allLength = len(file_name_list)
    while cnt < allLength:
        message = q.get()
        cnt += 1
        print("
Copying %s, Process Bar is:%d%%" % (message, (cnt / allLength) * 100), end="")
    print("Copy Done!")

if __name__ == "__main__":
    main()
    end = time.time()
    print("Time-consuming: %#.2fs" % (end-start))

在使用单进程拷贝文件之前，需要手动删除test-copy文件夹

# single-pool-copy.py
import time 
import os
import shutil
import random

start = time.time()

def copyFile(file_name, old_folder_name, new_folder_name):
    time.sleep(random.random())
    shutil.copyfile(old_folder_name + '\' + file_name, new_folder_name + '\' + file_name, )


def main():
    old_folder_name = input("Please input the folder name you want to copy: ")

    new_folder_name = old_folder_name + "-copy"
    os.mkdir(new_folder_name)

    file_name_list = os.listdir(old_folder_name)

    cnt = 0
    allLength = len(file_name_list)

    for file in file_name_list:
        copyFile(file, old_folder_name, new_folder_name)
        cnt += 1
        print("
Copying %s, Process Bar is:%d%%" % (file, (cnt / allLength) * 100), end="")        
    print("Copy Done!")

if __name__ == "__main__":
    main()
    end = time.time()
    print("Time-consuming: %#.2fs" % (end-start))