生成环境filebeat logstash配置模板

Posted 2020-10-31 mrhuangrui

filebeat logstash配置模板说明

为了让不同类型的日志记录到不同index，实现日志分类，需要更改默认的配置文件，ELK更新迭代速度很快，网上以前的文档适用于之前的版本

filebeat的docment_type配置项已经在6版本中弃用，请使用本文配置

filebeat配置模板

filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - /var/log/messages
    - /var/log/secure
    - /var/log/*.log
  fields: 
    log_topics: syslog

- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_access.log
  fields: 
    log_topics: nginx_access

- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_error.log       ##nginx错误日志位置
  fields: 
    log_topics: nginx_error               ##nginx错误日志注明类型(logstash为不同类型创建不同索引)

- type: log
  enabled: true
  paths:
    - /data/game/logs/dts_svc.log
    - /data/game/logs/dts_web.log
  fields: 
    log_topics: tomcat_svc_web
  multiline.pattern: '^\sINFO|^\sERROR|^\sDEBUG|^\sWARN'      ##将日志info,error,debug,warn开头的作为一行（用于java日志多行合并,也可以用时间为开头）
  multiline.negate: true
  multiline.match: after
  exclude_lines: ['^ INFO','^ DEBUG']                         ##排除info,debug开头的行
  include_lines: ["^ ERROR", "^ WARN"]                        ##将error,warn开头的行传给logstash

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:

output.logstash:
  hosts: ["localhost:5044"]

logstash配置模板

input {
    beats {
        port => 5044
        ssl => true
        ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
        ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
    }
}

output {
    if [fields][log_topics] == "nginx_access" {            ##按照type类型创建多个索引
        elasticsearch {
            hosts => ["192.168.1.146:9200"]
            index => "logstash-%{type}-%{+YYYY.MM.dd}_log"
            user => "elastic" 
            password => "QhQ0wS4G74JpKC7jRPJD"
        }

    }

    if [fields][log_topics] == "nginx_error" {            ##按照type类型创建多个索引
        elasticsearch {
            hosts => ["192.168.1.146:9200"]
            index => "logstash-%{[fields][log_topics]}-%{+YYYY.MM.dd}_log"
            user => "elastic" 
            password => "QhQ0wS4G74JpKC7jRPJD"
        }
    }

    if [fields][log_topics] == "tomcat_svc_web" {            ##按照type类型创建多个索引
        elasticsearch {
            hosts => ["192.168.1.146:9200"]
            index => "logstash-%{[fields][log_topics]}-%{+YYYY.MM.dd}_log"
            user => "elastic" 
            password => "QhQ0wS4G74JpKC7jRPJD"
        }
    }

    stdout { codec => rubydebug }
}