logstash 2.2以上版本,nginx 错误日志切割

Posted 2020-08-07 举个栗子

    网上nginx错误日期切分的版本各式各样,能用的没几个,踩过很多坑,特意记录下:

if [type] == "xx_app_nginx_error_log" {
        grok    {
          patterns_dir => "/etc/logstash/conf.d/patterns"
          match => { "message" => "%{NGINXERROR_1}" }
        }
        mutate {
          #避免日期报错
          gsub => ["nginxErrorDateTime","/","-"]
          rename=>["host", "nginxErrorHost"]
        }
        #切最后一段错误信息
        if [nginxErrorInfo] {
        #下面有些版本的logstash使用get会报错
            ruby{
              code => "
            new_event = LogStash::Event.new(Hash[event[‘nginxErrorInfo‘].split(‘, ‘).map{|l| l.split(‘: ‘)}])
            new_event.remove(‘@timestamp‘)
            event.append(new_event)"

            }
          }
　　　　　　#下面这段是用了IPIP库,不用的话可以去掉
          ipip {
            source => "client"
            target =>  "ipip"
          }
          grok {
            patterns_dir => "/etc/logstash/conf.d/patterns"
            match => {"request"=>"%{NGINXERROR_REQUEST}"}

          }
          #删除以下字段,减少重复内容
          mutate{
            remove_field => [ "message","nginxErrorInfo","request"]
          }
          }

grok详细信息

NGINXERROR_1 (?<nginxErrorDateTime>\d{4}/\d{2}/\d{2}\s+\d{2}:\d{2}:\d{2})\s+\[(?<nginxErrorType>\w+)\]\s+\S+:\s+\*\d+(?<nginxErrorMsg>[^,]+), (?<nginxErrorInfo>.*)$
NGINXERROR_REQUEST "%{WORD:method} %{URIPATH:urlPath}(?:\?(?<urlParams>[^ ]+))? HTTP/%{NUMBER:httpVersion}"

 输出:

 "@version": "1",
    "@timestamp": "2016-09-07T09:46:08.078Z",
    "path": "/to/path/nginx_error.log",
    "type": "xx_app_nginx_error_log",
    "nginxErrorDateTime": "2016-09-07 17:46:07",
    "nginxErrorType": "error",
    "nginxErrorMsg": " recv() failed (104: Connection reset by peer) while reading response header from upstream",
    "nginxErrorHost": "localhost",
    "client": "223.155.36.205",
    "server": "xxxxx",
    "upstream": "\"fastcgi://127.0.0.1:9000\"",
    "host": "\"localhost\"",
    "ipip": {
      "country": "中国",
      "province": "湖南",
      "city": "邵阳",
      "carrier": "电信"
    },
    "method": "GET",
    "urlPath": "/api/test.php",
    "urlParams": "action=is_remind&ajax=1&cookie=0",
    "httpVersion": "1.1"