U盘和电脑指定目录同步，修改后可以作为几台服务器之间的文件同步

Posted 2020-10-09

使用场景：
同步 电脑 和 U盘 的文件，文件的修改、删除、新增，只需要运行脚本就可以同步 电脑 和 U盘 指定的文件夹中的所有文件；
电脑 和 U盘 之间同步时候以文件内容最新的为准；
从 U 盘中删除文件，电脑的同步目录也会自动删除，反之相同，这里的删除只是将文件从同步目录移动到：D:\PyDiskRemoedFiles
在 U 盘新增的文件，电脑的同步目录也会自动新增，反之相同；

配置：
1、将电脑上的所有 USB 设备拔出；
2、执行脚本，该脚本会在当前用户家目录下生成配置文件：Udisk_Backup.txt
修改哦欸之文件，在配置文件中需要指定电脑的同步目录和U盘的同步目录
3、在 U盘 主目录下放入一个文件名为 py_udisk_identify.txt 的空文件；

备注：
1、不可以将整个电脑盘或整个 U盘 作为同步目录，需要新建一个文件夹来指定作为同步目录；
2、一旦执行在指定的同步目录下会生成一个文件：SyncCompareFile ，这个文件请不要随意删除；
3、如果遇到程序报错，请删除电脑和U盘同步目录下的 SyncCompareFile 后再次执行脚本；
4、该脚本不会自动删除空的文件夹；
5、如果是自己在 U盘 或者 电脑对应的备份目录下删除的文件会被自动保存到：D:\PyDiskRemoedFiles

 

================================================================================================================================================================

#coding:utf-8
import os
import filecmp
import time
import datetime
import string
import pickle
import copy
import shutil
import re

#this script is use for windows files and USBdisk files synchronization
#2017年10月5日00:28:57
#author : allen

#Describe ::
#when the first use this script, you should remove all usb disk device , run the script to create config file
#config file in current user home directory ".Udisk_Backup" , this file saved computer all disk nodes name

# 使用场景：
# 同步 电脑 和 U盘 的文件，文件的修改、删除、新增，只需要运行脚本就可以同步 电脑 和 U盘 指定的文件夹中的所有文件；
# 电脑 和 U盘 之间文件内容以最新的为准；
#
#
# 配置：
# 1、将电脑上的所有 USB 设备拔出；
# 2、执行脚本，该脚本会在当前用户家目录下生成配置文件：Udisk_Backup.txt
#    修改哦欸之文件，在配置文件中需要指定电脑的同步目录和U盘的同步目录
# 3、在 U盘 主目录下放入一个文件名为 py_udisk_identify.txt 的空文件；
#
# 备注：
# 1、不可以将整个电脑盘或整个 U盘 作为同步目录，需要新建一个文件夹来指定作为同步目录；
# 2、一旦执行在指定的同步目录下会生成一个文件：SyncCompareFile ，这个文件请不要随意删除；
# 3、如果遇到程序报错，请删除电脑和U盘同步目录下的 SyncCompareFile 后再次执行脚本；
# 4、该脚本不会自动删除空的文件夹；

#global variable
ComputerConfFileName = None                             #in user home directory config file name "Udisk_Backup.txt"
UsbConfFileName = ‘py_udisk_identify.txt‘               #identify effective usb device config file name
ComputerSyncFolderKey = ‘ComputerSyncFolderKey‘          #home directory config file assign computer sync dir path
UsbSyncFolderKey = ‘UsbSyncFolderKey‘                    #home directory config file assign usb device sync dir path
SyncCompareFile = ‘SyncCompareFile‘                      #in each device directory saved the device files info
BackupDir = os.path.join(‘D:‘, ‘PyDiskRemoedFiles‘)


#get path files list
def get_dir_list(path, ig_files=None):
    files = os.listdir(path)
    if ig_files:
        for e_ig in ig_files:
            if e_ig in files:
                files.pop(files.index(e_ig))
    files = [os.path.join(path, f) for f in files]
    if_dir = [os.path.isdir(f) for f in files]
    return files, if_dir

#move file to backup
def move_file(src):
    global BackupDir
    date = datetime.datetime.now()
    time_stamp = ‘（%s_%s_%s_%s_%s）‘ % (date.year, date.month, date.day, date.hour, date.minute)
    sub_dir = ‘%s_%s_%s‘ % (date.year, date.month, date.day)
    path = os.path.join(BackupDir, sub_dir)
    base_n = os.path.basename(src)
    if not os.path.exists(path):
        os.makedirs(path)
    f_n = os.path.splitext(base_n)
    f_n = "%s_%s_%s" % (f_n[0], time_stamp, f_n[-1])
    f_n = os.path.join(path, f_n)
    shutil.move(src, f_n)


#get computer current disk nodes
def get_disks_info():
    ident_str = string.ascii_uppercase
    primary_nodes = []
    for node in ident_str:
        try:
            node = ‘%s:‘ % node
            os.stat(node)
            primary_nodes.append(node)
        except (FileNotFoundError, PermissionError) as e:
            pass
    return primary_nodes


#copy the file from path_1 to path_2
def copy_file(org_path, targ_p):
    targ_p_d = os.path.dirname(targ_p)
    if not os.path.exists(targ_p_d):
        os.makedirs(targ_p_d)
    try:
        shutil.copy(org_path, targ_p)
    except PermissionError:
        print(‘同步的文件已经被打开，请关闭文件后再次执行该脚本‘)
        time.sleep(100)



#compare two files if common and the newest will cover the old one
def compare_file(f_1, f_2):
    try:
        if not filecmp.cmp(f_1, f_2):
            f_1_mt = os.stat(f_1).st_mtime
            f_2_mt = os.stat(f_2).st_mtime
            if f_1_mt > f_2_mt:
                os.remove(f_2)
                shutil.copy(f_1, f_2)
            elif f_1_mt <  f_2_mt:
                os.remove(f_1)
                shutil.copy(f_2, f_1)
    except FileNotFoundError:
        pass


#get set meta data
def get_set_d(key_s, list_1, list_2):
    key_s = key_s.replace(‘\\‘, ‘/‘)
    list_1.extend(list_2)
    list_d = ‘?‘.join(list_1)
    list_d = list_d.replace(‘\\‘, ‘/‘)
    p_str = r‘[^?]*?%s‘ % key_s
    pattern = re.compile(p_str)
    res = pattern.findall(list_d)
    return res[0].replace(‘/‘, ‘\\‘)




#replace the set data assign char generate new char
def replace_set(set_data, old, new):
    result = set()
    while True:
        try:
            each = set_data.pop()
            result.add(each.replace(old, new))
        except KeyError:
            break
    return result


#use pickle save dict data
def save_files_info(f_n, in_d, key):
    """
    :param f_n: file name
    :param in_d: input data
    :param key: history_all_files, current_files
    :return:
    """
    in_d = set(in_d)
    old_cur_d = None
    if os.path.exists(f_n):
        with open(f_n, ‘rb‘) as fp:
            all_d = pickle.load(fp)
            f_d = all_d.get(key, set())
        if key == ‘history_all_files‘:
            dif_d = in_d - f_d
            f_d.update(dif_d)
            all_d[key] = f_d
        else:
            old_cur_d = all_d.get(key, set())
            all_d[key] = in_d
        with open(f_n, ‘wb‘) as fp:
            pickle.dump(all_d, fp)
        return old_cur_d
    else:
        with open(f_n, ‘wb‘) as fp:
            pickle.dump({key: in_d}, fp)



#read pickle file, the file content is a dict
def read_pickle(f_n, key):
    try:
        with open(f_n, ‘rb‘) as fp:
            data = pickle.load(fp)
        data = data.get(key, set())
        return data
    except FileNotFoundError:
        return set()

#make directorys
def mk_dirs(p):
    if not os.path.exists(p):
        os.makedirs(p)

#recursion assign path get all files list
class BaseClass(object):
    def __init__(self):
        self.all_files = []

    def get_all_files(self, path, ig_files=None):
        files, if_dir = get_dir_list(path, ig_files)
        while any(if_dir):
            for index, judge in enumerate(if_dir):
                if_dir.pop(index)
                f = files.pop(index)
                if not judge:
                    self.all_files.append(f)
                else:
                    self.get_all_files(f)
        else:
            self.all_files.extend(files)


#create init conf file in current user home dir
class InitConfig(object):
    def __init__(self):
        self.home_path = os.path.join(‘C:‘, os.environ[‘HOMEPATH‘])     #get current user home dir
        self.primary_disks = self.local_d_record()                      #get computer primary folders

    def local_d_record(self):
        global ComputerConfFileName, ComputerSyncFolderKey, UsbSyncFolderKey
        conf_n = os.path.join(self.home_path, ‘%s.txt‘ % os.path.basename(__file__).split(‘.‘)[0])
        ComputerConfFileName = conf_n
        if not os.path.exists(conf_n):
            primary_nodes = get_disks_info()
            with open(conf_n, ‘w‘) as fp:
                fp.write(‘%s\n\n#在等号后面填写自己的电脑的同步的文件夹，不能指定某个盘作为同步，一定需要同步文件夹\n%s=\n\n#在等号后面填写自己的U盘文件夹，不能指定整个 U盘，可以指定某个文件夹作为同步\n%s=‘ % (str(primary_nodes), ComputerSyncFolderKey, UsbSyncFolderKey))
            return primary_nodes
        else:
            with open(conf_n, ‘r‘) as fp:
                try:
                    primary_nodes = eval(fp.readline().replace(‘ ‘, ‘‘))
                    return primary_nodes
                except SyntaxError:
                    print(‘请手动删除该文件后再次执行脚本 :: %s‘ % ComputerConfFileName)



class SyncDir(BaseClass):
    def __init__(self):
        super().__init__()
        self.m_stor_dev = self.get_usb_driver()             #get config file portable storage device lists
        self.main()


    #get all can use storage device, this step include identify effective usb device
    def get_usb_driver(self):
        usb_devices = []
        try:
            usb_disk = set(get_disks_info()) - set(InitConfig().primary_disks)          # get usb disk driver letter
            if not usb_disk:
                print(‘请按照以下步骤配置：\n1、把这个文件删除（如果没有就不用删除） ：%s‘ % ComputerConfFileName)
                print(‘2、把掉电脑上的所有 U 盘拔掉后再次执行脚本并修改该文件指定电脑和 U 盘目录：%s‘ % ComputerConfFileName)
                print(‘3、在你需要使用的 U盘 中放入一个文件名为 “py_udisk_identify.txt” 的文件，该文件用于识别需要同步的 U盘‘)
                return False
            else:
                while True:
                    try:
                        usb_id = usb_disk.pop()
                        usb_id_f = os.path.join(usb_id, UsbConfFileName)
                        if os.path.exists(usb_id_f):
                            usb_devices.append(usb_id)
                        else:
                            print(‘没能找到文件 ：%s\n所以认为 %s 盘 不是用来同步的移动存储设备，如果需要使用该设备同步请在该设备下放入一个文件名为的空文件 ：%s‘ % (usb_id_f, usb_id, UsbConfFileName))
                    except KeyError:
                        return usb_devices
        except TypeError:
            pass

    #read user directory config file
    def get_conf_info(self):
        global ComputerConfFileName
        result = {}
        with open(ComputerConfFileName, ‘r‘) as fp:
            data = fp.read()
        data = [d for d in data.split(‘\n‘) if ‘=‘ in d]
        for i in data:
            c = i.split(‘=‘)
            result[c[0]] = c[1].replace(‘ ‘, ‘‘)
        return result

    #get computer and usb storage dir
    def get_device_dir(self):
        _dir = self.get_conf_info()
        com_dir = _dir[‘ComputerSyncFolderKey‘]
        if not _dir[‘UsbSyncFolderKey‘]:
            usb_dir = self.get_usb_driver()
        else:
            usb_dir = [os.path.join(dr, _dir[‘UsbSyncFolderKey‘]) for dr in self.get_usb_driver()]
        return com_dir, usb_dir

    #according computer and usb assgin directory build all files info
    def main(self):
        global SyncCompareFile
        com_dir, usb_dir = self.get_device_dir()
        usb_dir_bak = copy.deepcopy(usb_dir)
        usb_dir.append(com_dir)
        old_cur_data = {}
        for each_dir in usb_dir:
            f_n = os.path.join(each_dir, SyncCompareFile)
            mk_dirs(each_dir)
            self.get_all_files(each_dir, ig_files=[SyncCompareFile,])
            save_files_info(f_n=f_n, in_d=self.all_files, key=‘history_all_files‘)
            old_cur_d = save_files_info(f_n=f_n, in_d=self.all_files, key=‘current_files‘)
            old_cur_data[each_dir] = old_cur_d
            self.all_files = []
        com_files = read_pickle(os.path.join(com_dir, SyncCompareFile), key=‘current_files‘)
        com_files_bak = copy.deepcopy(com_files)
        com_files_pure = replace_set(com_files_bak, com_dir + os.sep, ‘‘)
        for u_dir in usb_dir_bak:
            u_files = read_pickle(os.path.join(u_dir, SyncCompareFile), key=‘current_files‘)
            u_del_files = old_cur_data.get(u_dir, set()) - u_files                      #deleted computer files
            u_del_files_bak = copy.deepcopy(u_del_files)
            u_del_files_pure = replace_set(u_del_files_bak, u_dir + os.sep, ‘‘)

            c_del_files = old_cur_data.get(com_dir, set()) - com_files                  #delted usb files
            c_del_files_bak = copy.deepcopy(c_del_files)
            c_del_files_pure = replace_set(c_del_files_bak, com_dir + os.sep, ‘‘)
            u_files_bak = copy.deepcopy(u_files)
            u_files_pure = replace_set(u_files_bak, u_dir + os.sep, ‘‘)
            diff_f = com_files_pure ^ u_files_pure                           # difference set

            #delete computer file
            while True:
                try:
                    e_u_del_file = u_del_files_pure.pop()
                    f_n = os.path.join(com_dir, e_u_del_file)
                    try:
                        move_file(f_n)
                    except FileNotFoundError:
                        pass
                    diff_f.remove(e_u_del_file)
                except KeyError:
                    break

            #delete usb file
            while True:
                try:
                    e_c_del_file = c_del_files_pure.pop()
                    f_n = os.path.join(u_dir, e_c_del_file)
                    try:
                        move_file(f_n)
                    except FileNotFoundError:
                        pass
                    diff_f.remove(e_c_del_file)
                except KeyError:
                    break

            #process new add file
            while True:
                try:
                    e_dif_d = diff_f.pop()
                    abs_c_f_diff = os.path.join(com_dir, e_dif_d)
                    abs_u_f_diff = os.path.join(u_dir, e_dif_d)
                    if not os.path.exists(abs_c_f_diff):
                        copy_file(org_path=abs_u_f_diff, targ_p=abs_c_f_diff)
                    elif not os.path.exists(abs_u_f_diff):
                        copy_file(org_path=abs_c_f_diff, targ_p=abs_u_f_diff)
                except KeyError:
                    break

            #compare exists file and new will cover olders
            comm_set = com_files_pure & u_files_pure
            while True:
                try:
                    e_com_d = comm_set.pop()
                    abs_c_f_com = os.path.join(com_dir, e_com_d)
                    abs_u_f_com = os.path.join(u_dir, e_com_d)
                    compare_file(abs_c_f_com, abs_u_f_com)
                except KeyError:
                    break




if __name__ == ‘__main__‘:
    print(‘=‘*78)
    print(‘==如果看到提示信息，请按照提示信息操作后，并关闭这个命令窗口然后再次双击执行==‘)
    print(‘=‘*78)

    try:
        SyncDir()
    except Exception:
        print(‘程序运行出现了错误，很可能是因为 SyncCompareFile 这个文件被删除，请不要删除这个文件\n现在再次执行就可以了‘)
        time.sleep(100)
    print(‘程序执行完成！‘)
    time.sleep(100)