多维度分析统计nginx访问日志
Posted
tags:
篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了多维度分析统计nginx访问日志相关的知识,希望对你有一定的参考价值。
nginx日志访问量,多维度统计,__fields表示每条日志以空格分隔后索引对应的字段名称。
如0表示第一个字段客户端IP(client_ip),要统计其它字段,只需在这里添加即可。
## 直接上代码
# -*- coding:utf-8 -*- import sys try: import json except ImportError: import simplejson as json class AnalysisNginxLog(object): ‘‘‘ analysis nginx proxy access log proxy log format: log_format proxy ‘$remote_addr - $host [$time_local] "$request" $status‘ ‘ $body_bytes_sent "$http_referer" ‘ ‘"$http_user_agent" "$http_x_forwarded_for/$upstream_addr" "$request_time/$upstream_response_time/$upstream_status/$userId" "$xm_logid"‘; ‘‘‘ __fields = [ (0, ‘client_ip‘), (2, ‘domain‘), (3, ‘access_time‘), # (4, ‘time_zone‘), (5, ‘http_method‘), (6, ‘uri‘), (7, ‘http_version‘), (8, ‘request_status‘), (9, ‘request_body_size‘) ] __filter_args = [ ‘uri‘, ‘access_time‘ ] def __init__(self, logfile): self.logfile = logfile self.data = [] self._load_data() def _open(self): with open(self.logfile) as fp: while True: line = fp.readline().strip() if not line: break yield line def get_data(self): self._load_data() return self.data def _filter1(self, data): return data.strip().strip(‘#‘).strip(‘"‘) def _filter2(self, data): if not isinstance(data, dict): raise Exception(‘Invalid Parameters, must be type of dict!‘) for key, value in data.items(): if key not in self.__filter_args: continue if ‘?‘ in value: v = value.split(‘?‘)[0] data[key] = v elif ‘[‘ in value: data[key] = value.strip(‘[‘) return data def _load_data(self): for line in self._open(): li = [x.strip() for x in line.split()] dict_tmp = {} for i, field in self.__fields: try: dict_tmp[field] = self._filter1(li[i]) except IndexError as e: print(‘Error: {}, at {}‘.format( e, line )) dict_tmp[field] = None self.data.append(self._filter2(dict_tmp)) def counter_by_field(self, field, top): fields = [x[1] for x in self.__fields] if field not in fields: raise Exception(‘Invalid Parameters!‘) result = {} for log in self.data: if log[field] in result: result[log[field]] += 1 else: result[log[field]] = 1 result = sorted(result.items(), key=lambda d: d[1], reverse=True)[:top] print(json.dumps(result, indent=2)) return result def _help(): print(‘Usage: %s [log_file] [colums] [topN] ‘ % sys.argv[0]) sys.exit() def counter_log(logfile, colums, top=10): nginx = AnalysisNginxLog(logfile) cols = colums.split(‘,‘) for col in cols: print(‘{c} [{col}] Top {top} {c}‘.format(c=‘-‘ * 30, col=col, top=top)) nginx.counter_by_field(col, int(top)) if __name__ == ‘__main__‘: if len(sys.argv) <= 2: _help() counter_log(*sys.argv[1:])
## 运行结果
# > python analysis_nginx_proxy_log.py Usage: analysis_nginx_proxy_log.py [log_file] [colums] [topN]
> python analysis_nginx_proxy_log.py pss_00-06.log uri,client_ip 3 ------------------------------ [uri] Top 3 ------------------------------ [ [ "/api/pss/GetBatchSnDetails", 2298 ], [ "/api/xmss/CheckSubHome", 1456 ], [ "/api/xmss/GetPhyStock", 1299 ] ] ------------------------------ [client_ip] Top 3 ------------------------------ [ [ "10.108.38.18", 2150 ], [ "10.114.135.253", 1403 ], [ "10.104.3.11", 1253 ] ]
本文出自 “JUST DON'T GIVE UP!” 博客,请务必保留此出处http://zliang90.blog.51cto.com/3001993/1964737
以上是关于多维度分析统计nginx访问日志的主要内容,如果未能解决你的问题,请参考以下文章
Linux脚本练习之script082-nginx日志分析之统计访问3次以上的IP
Linux脚本练习之script082-nginx日志分析之统计访问3次以上的IP