Nginx访问日志和错误日志的拆分（Logstash）

Posted 2020-10-28 zhuhaiqing

>> from zhuhaiqing.info

input {
  file {
    type =>> "nginx-access"  
    path =>> [ "/var/log/nginx/access.log" ]
    tags =>> [ "nginx","access"]
    start_position =>> beginning
  }
  file {
    type =>> "nginx-error" 
    path =>> [ "/var/log/nginx/error.log" ]
    tags =>> [ "nginx","error"]
    start_position =>> beginning
  }
}
filter {
  if [type] == "nginx-access" {
    grok{
      match =>> ["message","%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"]
    }
    date{
      match =>> ["time","dd/MMM/yyyy:HH:mm:ss Z"]
      target =>> "logdate"
    }
    ruby{
      code =>> "event.set(‘logdateunix‘,event.get(‘logdate‘).to_i)"
    }
  } 
      else if [type] == "nginx-error" { 
    grok {
      match =>> [
        "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:,\s{1,}client:\s{1,}(?<client_ip>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:client_ip})?(?:, referrer: \"%{URI:referrer})?",
        "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}%{GREEDYDATA:err_message}"
        ]
    }
    date{
      match=>>["time","yyyy/MM/dd HH:mm:ss"]
      target=>>"logdate"
    }
    ruby{
      code =>> "event.set(‘logdateunix‘,event.get(‘logdate‘).to_i)"
    }
     }
}
output{
  elasticsearch{
    hosts =>> ["192.168.100.10:9200"]
    index =>> "logstash-nginx-%{+YYYY.MM.dd}"
  }
}