日志导入金融云oss

Posted 2020-11-12 linhankbl

#! /usr/bin/env python
# ! -*- coding:utf8 -*-

from __future__ import print_function
import oss2
import os, sys
import datetime
import socket
from pymongo import MongoClient

hostName = socket.gethostname()
today = datetime.date.today()
yesterday = today + datetime.timedelta(days=-1)
logday = str(yesterday)

auth = oss2.Auth(‘LTAIz5gSKA8bhvG3‘, ‘ZVbFGrLIZfn9sYTuRX0nTDSpkrdsMb‘)
imgs_bucket = oss2.Bucket(auth, ‘http://oss-cn-hzfinance.aliyuncs.com‘, ‘logspace‘)
# 本地资源路径
apps = (‘/home/admin/logs/biportal‘, ‘/home/admin/logs/zdal‘)
application = ‘cmdb‘   #应用名

uploads_dirs = [app for app in apps]
local_res_files = []
log_files = {}


# 文件归类,获取要上传的本地日志文件的所有文件的绝对路径
def getLocalFiles(dir):
    if os.path.exists(dir):
        for res in os.listdir(dir):
            ab_dir = dir + res
            if os.path.isfile(ab_dir):
                if logday in res:
                    log_files[res] = ab_dir
            if os.path.isdir(ab_dir):
                getLocalFiles(ab_dir)


# 上传文件
def putFileToBucket():
    for key, value in log_files.items():
        result = imgs_bucket.put_object_from_file(application + ‘/‘ + hostName + ‘/‘ + logday + ‘/‘ + key, value,headers={‘Content-Type‘: ‘text/plain; charset=utf-8‘}, progress_callback = percentage)
        now_time = str(datetime.datetime.now())
        print(value)
        conndb(hostName,key,value,now_time,application + ‘/‘ + hostName + ‘/‘ + logday)
        if (os.path.exists(value)):
            os.remove(value)



# 起始程序
def start(dirs):
    for dir in dirs:
        getLocalFiles(dir)


# 进度条功能
def percentage(consumed_bytes, total_bytes):
    if total_bytes:
        rate = int(100 * (float(consumed_bytes)) / (float(total_bytes)))
        print(‘\r{0}%‘.format(rate), end=‘‘)
        sys.stdout.flush()

#信息输出mongo
def conndb(servername,filename,filedir,updatetime,updatedir):
    conn = MongoClient(‘IP‘, 27017)
    db = conn.logdb
    my_set = db.logfile
    my_set.insert({"servername":servername,"filename":filename,"filedir":filedir,"updatetime":updatetime,"updatedir":updatedir})


if __name__ == "__main__":
    start(uploads_dirs)
    putFileToBucket()