logstash 系统日志和tomcat日志收集

Posted 2022-01-15 小怪獣55

文档：​​https://www.elastic.co/guide/en/logstash/6.8/plugins-inputs-file.html​​

1.对系统日志的采集

root@ubuntu:~# vim /etc/logstash/conf.d/system-log.conf

input 
 file
   path => "/var/log/syslog"
   start_position => "beginning"
   stat_interval => 3
   type => "syslog"
   


output 
  if [type] =="syslog"
     elasticsearch
       hosts => ["192.168.47.106:9200"]
       index => "linux47-syslog-%+YYYY.MM.dd"
       
    file 
       path => "/tmp/syslog.txt"
     
  

检测配置文件

/usr/share/logstash/bin/logstash  -f  /etc/logstash/conf.d/system-log.conf -t

守护进程方式启动

/usr/share/logstash/bin/logstash  -f  /etc/logstash/conf.d/system-log.conf

2.收集tomcat日志

收集Tomcat服务器的访问日志以及Tomcat错误日志进行实时统计，在kibana页面进行搜索展现，每台Tomcat服务器要安装logstash负责收集日志，然后将日志转发给 elasticsearch 进行分析，在通过 kibana 在前端展现

jdk及tomcat安装，参考：​​https://blog.51cto.com/u_14814545/4898618​​

tomcat修改配置文件,使产生的日志为json格式

root@ubuntu:/data/tomcat# vim conf/server.xml 
----------------------------------------------------------------------------------------
prefix="localhost_access_log" suffix=".log"
pattern=""client":"%h","client user":"%l","authenticated":"%u","access time":"%t","method":"%r","status":"%s","send bytes":"%b","Query?string":"%q","partner":"%Refereri","Agent version":"%User-Agenti""/>
----------------------------------------------------------------------------------------

root@ubuntu:/data/tomcat# rm -rf logs/*
root@ubuntu:/data/tomcat# ./bin/catalina.sh start

json格式验证：​​http://www.kjson.com/​​

vim /etc/logstash/conf.d/tomcat.conf

input
 file
  path => "/data/apache-tomcat-8.5.39/logs/localhost_access_log.*.log"
#   path => "/data/apache-tomcat-8.5.39/logs/localhost_access_log.2022-01-14.log"
   start_position => "beginning"
   stat_interval => 3
   type => "tomcat-access-log"
   codec => "json"
   

output
if [type] == "tomcat-access-log"
    elasticsearch
    hosts => ["192.168.47.106:9200"]
    index => "tomcatlog-%+YYYY.MM.dd"
     
#  file 
#     path => "/tmp/tomcat.log" 
#     
#   
#stdout
#    codec => "rubydebug"
#   
 

测试

root@ubuntu:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat.conf

通过python脚本获取日志中的IP

status_200 = []
status_404 = []
with open("/data/tomcat/logs/localhost_access_log.2022-01-14.log") as f:
    for line in f.readlines():
        line = eval(line)
        #print(line.get("clientip"))
        if line.get("status") == "200":
            status_200.append(line.get)
        elif line.get("status") == "404":
            status_404.append(line.get)
        else:
            print("状态码 ERROR")
f.close()
print("状态码200的有-->:",len(status_200))
print("状态码404的有-->:",len(status_404))

参考文档：​​https://opsblogs.cn/?p=746​​