python脚本实现访问日志合并

Posted 2020-09-29

网易云对象存储的访问日志默认按小时生成一个日志文件，不利于统计当日整体数据，于是考虑将多个日志文
件合并到一个文件中

#!/usr/bin/evn python
# -*- coding:utf-8 -*-
import nos
import time
import sys
import os
import re

access_key = "54175b3a0a544f07a81d618719d44a0a"
secret_key = "513b131cd66d46f6bc5a3e5ca5784780"
bucket = "access-log"
date = time.strftime(‘%Y-%m-%d‘,time.localtime(time.time() - 24*60*60))
prefix = ‘aoshu‘+date
client = nos.Client(access_key, secret_key)
def get_object_lists():
    try:
        object_lists = client.list_objects(bucket, prefix=prefix)
        file_lists = []
        for object_list in object_lists["response"].findall("Contents"):
            file_lists.append(object_list.find("Key").text)
            print object_list.find("Key").text
        if not file_lists:
            print "不存在该时间段内任何日志文件,退出程序..."
            sys.exit(1)
        else:
            #print file_lists
            return file_lists
    except nos.exceptions.ServiceException as e:
        print (
            "ServiceException: %s\n"
            "status_code: %s\n"
            "error_type: %s\n"
            "error_code: %s\n"
            "request_id: %s\n"
            "message: %s\n"
        ) % (
            e,
            e.status_code,   
            e.error_type,   
            e.error_code, 
            e.request_id,
            e.message
       )
    except nos.exceptions.ClientException as e:
        print (
            "ClientException: %s\n"
            "message: %s\n"
         ) % (
            e,
            e.message 
        )
def save_log():
    objects_lists = get_object_lists()
    log_file_name="nos.log"
    with open(log_file_name,‘a‘) as f:
        for object in objects_lists:
            try:
                result = client.get_object(bucket,object)
                f.write(result.get("body").read())
            except nos.exceptions.ServiceException as e:
                print (
                "ServiceException: %s\n"
                "status_code: %s\n"
                "error_type: %s\n"
                "error_code: %s\n"
                "request_id: %s\n"
                "message: %s\n"
                 ) % (
                 e,
                 e.status_code, 
                 e.error_type,  
                 e.error_code, 
                 e.request_id,
                 e.message 
                        )
            except nos.exceptions.ClientException as e:
                print (
                "ClientException: %s\n"
                "message: %s\n"
                 ) % (
                 e,
                 e.message 
                  )
    return log_file_name

if __name__==‘__main__‘:
    save_log()