python 实时分析日志

Posted 2020-09-14

#!/usr/local/bin/python3
# coding:utf-8

# ====================================================
# Author: chang - EMail:[email protected]
# Last modified: 2017-4-28
# Filename: nginxanalysis.py
# Description: real time analysis nginx log,base time, os
# blog:http://www.cnblogs.com/changbo
# ====================================================

"""
需求：每隔1分钟读取nginx日志文件
"""


import time
import os


# write log offset
def writeoffset(number):
    with open(‘offset.txt‘, ‘w+‘) as f3:
        f3.write(number)
        f3.flush()


# get log offset
def getoffset():
    with open(‘offset.txt‘) as f2:
        offset = f2.readline()
        return offset


# online analysis log
def analysislog():
    with open(‘access.log‘) as f1:
        while True:
            # get offset
            lastoffset = getoffset()
            # jump the Specify log line
            f1.seek(int(lastoffset))
            # 获取该行偏移量
            where = f1.tell()
            line = f1.readline()
            writeoffset(str(where))
            if not line:
                time.sleep(10)
                f1.seek(where)
            else:
                # 处理该行，并获取改行的偏移量且写入文件
                print(line)
                nowoffset = f1.tell()
                writeoffset(str(nowoffset))

if __name__ == ‘__main__‘:
    if not os.path.exists(‘offset.txt‘):
        with open("offset.txt", ‘w‘) as f:
            f.write(‘0‘)

    analysislog()

 

模拟日志脚本

#!/usr/local/bin/python3
# coding:utf-8

# ====================================================
# Author: chang - EMail:[email protected]
# Last modified: 2017-4-28
# Filename: testlog.py
# Description: create log files,base time, os
# blog:http://www.cnblogs.com/changbo
# ====================================================

import time
import os


def writecount(number):
    with open(‘count.txt‘, ‘w+‘) as f1:
        f1.write(number)
        f1.flush()


def getcount():
    with open(‘count.txt‘) as f2:
        line = f2.readline()
        return line


def writetest():
    with open(‘access.log‘, ‘a+‘) as f:
        count = getcount()
        count = int(count)
        while True:
            nowtime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
            eglogtxt = "this is %d line" % count
            time.sleep(3)
            f.write(nowtime + " " + eglogtxt + ‘\\n‘)
            count += 1
            f.flush()
            writecount(str(count))

if __name__ == ‘__main__‘:
    if not os.path.exists(‘count.txt‘):
        with open("count.txt", ‘w‘) as f:
            f.write(‘1‘)

    writetest()

 

模拟日志切割过程中初始化脚本参数

cp access.log access20170408.log  && echo > access.log && echo ‘0‘> offset.txt

 

 

成功

End!