定时日志清理

Posted 2021-02-02 sea-stream

 

作用：

针对的log目录是家目录的.ros文件夹，目录结构是/home/xxx/.ros/log/latest

.ros中除了log文件夹，其他缓存文件全部清除

然后就是log文件夹里面，这里面就是真正的log文件了，会有很多文件夹，每次重启，都会在log目录生成日志文件夹如：swswdedefraa-323232-wswdededeede,该文件夹会被打包，放到/opt/xxx/logs中，还有一些.log文件，.log文件全部清除

首先来说打包，默认情况下，我们保留/home/xxx/.ros/log/目录中三个最新的日志文件夹，其他的会被打包到/opt/xxx/logs，并且删除文件夹，打包后要检查是否大于50M，如果大于50M就会分割为20M的文件分段存储，

接着 判断/home/xxx/.ros/log/的目录大小，如果大于800M，就会只保留一个最新的文件夹，其他的全部压缩，删除

还要检查/opt/xxx/logs，如果大于1G，那就删除旧的压缩包，直到他的大小小于1G。

加个定时任务，定时执行该脚本，就可以做到日志自动备份清除。

#-*- encoding=utf8 -*-
import time 
import os
import sys
import math
import zipfile
import tarfile
import commands
from os.path import join, getsize
fpath="/home/xxx/"
targetpath="/opt/xxx/logs/"
chunksize = 20*(1024**2)
log_path=fpath+".ros/"+"log/"

def get_FileSize(filePath):
    fsize = os.path.getsize(filePath)
    #fsize = fsize/float(1024*1024)
    #return round(fsize,5)
    return fsize
def getdirsize(dirpath):
    size = 0
    for root, dirs, files in os.walk(dirpath):
        size += sum([getsize(join(root, name)) for name in files])
    return size
def get_time_stamp(ct):
    local_time = time.localtime(ct)
    data_head = time.strftime("%Y-%m-%d--%H:%M:%S", local_time)
    data_secs = (ct - int(ct)) * 1000
    time_stamp = "%s:%03d" % (data_head, data_secs)
    return time_stamp
def fileTime(file):
    #time.ctime(os.path.getatime(file))
    return [
        os.path.getatime(file),
        os.path.getctime(file),
        os.path.getmtime(file)
    ]
def make_targz(output_filename, source_dir):
    with tarfile.open(output_filename, "w:gz") as tar:
        tar.add(source_dir, arcname=os.path.basename(source_dir))
def make_targz_one_by_one(output_filename, source_dir):
    tar = tarfile.open(output_filename,"w:gz")
    for root,dir,files in os.walk(source_dir):
        for file in files:
            pathfile = os.path.join(root, file)
            tar.add(pathfile)
    tar.close()

def del_tar(tar_list):
    if(getdirsize(targetpath)<=1024*(1024**2)):
        return 1
    else:
        i=tar_list.pop()
        print("delete "+i[0]+" successfully!")
        os.remove(targetpath+i[0])
        return del_tar(tar_list)

def split(fromfile,todir,chunksize):
    partnum = 0
    file_name=os.path.basename(fromfile)
    inputfile = open(fromfile,‘rb‘)#open the fromfile
    while True:
        chunk = inputfile.read(chunksize)
        if not chunk:             #check the chunk is empty
            break
        partnum += 1
        filename = os.path.join(todir,(‘{0}-({1}).tar.gz‘.format(file_name.split(".")[0],partnum)))
        fileobj = open(filename,‘wb‘)#make partfile
        fileobj.write(chunk)         #write data into partfile
        fileobj.close()
    return partnum
file_dic={}
filetwo_dic={}
tar_dic={}
sorted_list=[]
if(os.path.exists(fpath+".ros")):
    file_list=os.listdir(fpath+".ros")
    if(file_list):
        if("log" in file_list):
            file_list.remove("log")
        if(file_list):
            for j in file_list:
                status, output = commands.getstatusoutput("sudo rm -rf "+fpath+j)
                if(not status):
                    print("delete .ros/ successfully!")

if(os.path.exists(log_path)):
    sh="sudo rm -rf "+fpath+".ros/log/*.log"
    status, output = commands.getstatusoutput(sh)
    if(not status):
        print("delete *.log successfully!")
    file_list=os.listdir(log_path)
    if(file_list):
        if("latest" in file_list):
            file_list.remove("latest")
        if(file_list):
            if(len(file_list)>3):
                for i in file_list:
                    file_dic[i]=fileTime(log_path+i)[2]
                ccc=sorted(file_dic.items(),key = lambda x:x[1],reverse = True)[3:]
                sorted_list=[x for x in ccc if os.path.isdir(log_path+x[0])]
                for j in sorted_list:
                    print("to tar...")
                    t_tar=targetpath+get_time_stamp(os.path.getmtime(log_path+j[0]))+".tar.gz"
                    if(os.path.exists(t_tar)):
                        os.remove(t_tar)
                        print("remove already exist file!")
                    print(t_tar)
                    try:
                        make_targz(t_tar,log_path+j[0])
                    except Exception as e:
                        print(e)
                    print("to delete file...")
                    status, output = commands.getstatusoutput("sudo rm -rf "+log_path+j[0])
                    print("delete ok!")
                    if(not status):
                        print("delete .ros/log successfully!")
                    if(get_FileSize(t_tar)>(50*(1024**2))):
                        try:
                            parts = split(t_tar,targetpath,chunksize)
                            print("split ok!")
                        except:
                            print(‘Error during split:‘)
                            print(sys.exc_info()[0],sys.exc_info()[1])
                        else:
                            print("DELETE old tar")
                            os.remove(t_tar)
                            print("delete success!!!")
            if(getdirsize(log_path)>=800*(1024**2)):
                if(len(file_list)==1):
                    pass
                else:
                    for i in file_list:
                        filetwo_dic[i]=fileTime(log_path+i)[2]
                    ccc=sorted(filetwo_dic.items(),key = lambda x:x[1],reverse = True)[1:]
                    sorted_list=[x for x in ccc if os.path.isdir(log_path+x[0])]
                    for j in sorted_list:
                        print("to tar...")
                        t_tar=targetpath+get_time_stamp(os.path.getmtime(log_path+j[0]))+".tar.gz"
                        if(os.path.exists(t_tar)):
                            os.remove(t_tar)
                            print("remove already exist file!")
                        print(t_tar)
                        try:
                            make_targz(t_tar,log_path+j[0])
                        except Exception as e:
                            print(e)
                        print("to delete file...")
                        status, output = commands.getstatusoutput("sudo rm -rf "+log_path+j[0])
                        print("delete ok!")
                        if(not status):
                            print("delete .ros/log successfully!")
                        if(get_FileSize(t_tar)>(50*(1024**2))):
                            try:
                                parts = split(t_tar,targetpath,chunksize)
                                print("split ok!")
                            except:
                                print(‘Error during split:‘)
                                print(sys.exc_info()[0],sys.exc_info()[1])
                            else:
                                print("DELETE old tar")
                                os.remove(t_tar)
                                print("delete success!!!")
if(os.path.exists(targetpath)):
    file_list=os.listdir(targetpath)
    if(file_list):
        if("server.log" in file_list):
            file_list.remove("server.log")
        if(file_list):
            for i in file_list:
                tar_dic[i]=fileTime(targetpath+i)[2]
            tar_list=sorted(tar_dic.items(),key = lambda x:x[1],reverse = True)
            if(del_tar(tar_list)):
                print("tar directory checked!")