python 用蟒蛇实现的日志流式分析

Posted 2021-05-09

# 用来产生日志以供测试
$ i=0; while true; do echo $i' hello' >> access.log; let i++; sleep 0.01; done;

# -*- coding: utf-8 -*-
'''
tail一个文件，做流式分析，可以用于日志输出的监控。
此处示例为每3秒输出统计得到的行数和字符数。
'''
import time
import subprocess
import datetime
from threading import Timer

filename = 'access.log'
interval = 3        # 间隔时间
result_init = {
    'chars': 0,
    'lines': 0,
}
result = dict(result_init)

def readfile(filename):
    f = subprocess.Popen(['tail','-c','0','-F',filename], stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    while True:
        line = f.stdout.readline()
        process(line)

def process(line):
    '''处理一行'''
    global result
    result['chars'] += len(line)
    result['lines'] += 1

def summary():
    global result
    print u'【{0:%Y-%m-%d %H:%M:%S}】{2} lines, {3} chars in {1}s'.format(
            datetime.datetime.now(),
            interval,
            result['lines'],
            result['chars'])
    result = dict(result_init)
    Timer(interval, summary, ()).start()

if __name__ == '__main__':
    print 'start to analysis...'
    Timer(0, readfile, (filename, )).start()
    Timer(interval, summary, ()).start()