Logstash解析Json array

Posted 2021-01-14 wenbo的博客

logstash解析json数组是一种常见的需求，我以网上一组数据为例来描述

我们的数据test.json内容如下：(此处我linux上的json文本需要是compact的)

{"type":"monitor","server":"10.111.222.333","host":"abc.de","bean":[{"name":"beanName1","reseted":"2015-06-05T15:10:00.192Z","method":[{"name":"getAllXY","count":5,"min":3,"max":5},{"name":"getName","count":4,"min":2,"max":4}]},{"name":"beanName2","reseted":"2015-06-05T15:10:00.231Z","method":[{"name":"getProperty","count":4,"min":3,"max":3}]},{"name":"beanName3","reseted":"2015-06-05T15:10:00.231Z"}]}

为了方便看清楚内容，我们format后查看：

{
"type": "monitor",
"server": "10.111.222.333",
"host": "abc.de",
"bean": [{
    "name": "beanName1",
    "reseted": "2015-06-05T15:10:00.192Z",
    "method": [{
      "name": "getAllXY",
      "count": 5,
      "min": 3,
      "max": 5
    },
    {
      "name": "getName",
      "count": 4,
      "min": 2,
      "max": 4
    }]
  },
  {
    "name": "beanName2",
    "reseted": "2015-06-05T15:10:00.231Z",
    "method": [{
      "name": "getProperty",
      "count": 4,
      "min": 3,
      "max": 3
    }]
  },
  {
    "name": "beanName3",
    "reseted": "2015-06-05T15:10:00.231Z"
  }]
}

我们可以看到bean字段下是一个json数组，解析这种json数组，我们需要借用logstash split filter plugin

测试：单纯地把bean字段下的json拆分出来

我的配置文件如下

input {
        file {
            path => "/usr/share/logstash/private.cond/split.json"
            codec => "json"
            start_position => "beginning"
            sincedb_path => "/dev/null"
        }
}
filter {
         json {
            source => "message"
         }
         split {
            field => "bean"
         }
}
 
output {
        stdout {
            codec => rubydebug
        }
}

我们得到如下输出结果

{
      "@version" => "1",
        "server" => "10.111.222.333",
          "type" => "monitor",
          "bean" => {
           "name" => "beanName1",
         "method" => [
            [0] {
                  "min" => 3,
                 "name" => "getAllXY",
                "count" => 5,
                  "max" => 5
            },
            [1] {
                  "min" => 2,
                 "name" => "getName",
                "count" => 4,
                  "max" => 4
            }
        ],
        "reseted" => "2015-06-05T15:10:00.192Z"
    },
          "path" => "/usr/share/logstash/private.cond/split.json",
    "@timestamp" => 2018-08-02T10:36:21.248Z,
          "host" => "abc.de"
}
{
      "@version" => "1",
        "server" => "10.111.222.333",
          "type" => "monitor",
          "bean" => {
           "name" => "beanName2",
         "method" => [
            [0] {
                  "min" => 3,
                 "name" => "getProperty",
                "count" => 4,
                  "max" => 3
            }
        ],
        "reseted" => "2015-06-05T15:10:00.231Z"
    },
          "path" => "/usr/share/logstash/private.cond/split.json",
    "@timestamp" => 2018-08-02T10:36:21.248Z,
          "host" => "abc.de"
}
{
      "@version" => "1",
        "server" => "10.111.222.333",
          "type" => "monitor",
          "bean" => {
        "reseted" => "2015-06-05T15:10:00.231Z",
           "name" => "beanName3"
    },
          "path" => "/usr/share/logstash/private.cond/split.json",
    "@timestamp" => 2018-08-02T10:36:21.248Z,
          "host" => "abc.de"
}

根据输出我们可以看到json数组被我单个拆分出来。