python 多进程拷贝文件目录

Posted 2021-02-15 code never lies

 1 # -*- coding: utf-8 -*-
 2 # @author: Tele
 3 # @Time  : 2019/04/02 下午 3:09
 4 # 借助shutil使用多进程拷贝文件夹
 5 # 使用进程池实现多进程时,使用的消息队列要使用multiprocessing.Manager().Queue()创建
 6 
 7 import time
 8 import re
 9 import os
10 import shutil
11 import multiprocessing
12 
13 
14 # 遍历文件夹
15 def walk_file(file):
16     file_list = list()
17     for root, dirs, files in os.walk(file):
18         # 遍历文件
19         for f in files:
20             file_list.append(f)
21     return file_list
22 
23 
24 # 计算文件数量
25 def get_file_count(dir):
26     return len(walk_file(dir))
27 
28 
29 def copy(src, target, queue):
30     target_number = 1
31     if os.path.isdir(src):
32         target_number = get_file_count(src)
33         shutil.copytree(src, target)
34     else:
35         shutil.copyfile(src, target)
36     # 将拷贝完成的文件数量放入队列中
37     queue.put(target_number)
38 
39 
40 def copy_dir(src, desc):
41     total_number = get_file_count(src)
42     # 分隔符检测
43     src = check_speator(src)
44     desc = check_speator(desc)
45     # print("src:",src)
46     # print("desc:",desc)
47 
48     file_dir_list = [src + "/" + i for i in os.listdir(src)]
49     if os.path.exists(desc):
50         shutil.rmtree(desc)
51     pool = multiprocessing.Pool(3)
52 
53     # 创建队列
54     queue = multiprocessing.Manager().Queue()
55 
56     # 一个文件/目录开启一个进程去拷贝
57     for f_name in file_dir_list:
58         target = desc + "/" + f_name[index_list("/", f_name)[1] + 1:]
59         # print(target)
60         # 创建target目录
61         parent_path = os.path.split(target)[0]
62         if not os.path.exists(parent_path):
63             os.makedirs(parent_path)
64         pool.apply_async(copy, args=(f_name, target, queue,))
65 
66     start = time.time()
67     pool.close()
68     #    pool.join()
69     count = 0
70     while True:
71         count += queue.get()
72         # 格式化输出时两个%输出一个%,不换行,每次定位到行首,实现覆盖
73         print("
拷贝进度为 %.2f %%" % (count * 100 / total_number), end="")
74         if count >= total_number:
75             break
76     end = time.time()
77     print()
78     print("耗时-----", (end - start), "s")
79 
80 
81 # 查找指定字符出现的全部索引位置
82 def index_list(c, s):
83     return [i.start() for i in re.finditer(c, s)]
84 
85 
86 # 检测目录结尾是否有 "/"
87 def check_speator(path):
88     if path.rindex("/") == len(path) - 1:
89         return path[0:path.rindex("/")]
90     return path
91 
92 
93 def main():
94     copy_dir("f:/ftp_mypc/", "e:/ftp_mypc/")
95 
96 
97 if __name__ == ‘__main__‘:
98     main()