日志收集三：rsyslog + elasticsearch + kibana

Posted 2020-11-14

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了日志收集三：rsyslog + elasticsearch + kibana相关的知识，希望对你有一定的参考价值。

rsyslog v8

配置文件语法改变较大，添加了很多其他输入输出模块，可以更好的进行扩展。
在v5的基础上，添加elasticsearch的发送。
v8配置参考：请添加链接描述

v8采集端完整配置

module(load="imuxsock") # provides support for local system logging (e.g. via logger command)
module(load="imklog")   # provides kernel logging support (previously done by rklogd)
module(load="omelasticsearch")
module(load="imfile" mode="inotify" readtimeout="5" )
#input(type="imfile" file="/var/log/zabbix_*.log" tag="C+eebo.test.test+CD" ruleset="fileinput" facility="local6" PersistStateInterval="100" startmsg.regex="^[[:digit:]]{5}:[[:digit:]]{8}:[[:digit:]]{6}.[[:digit:]]{3}" maxsubmitatonce="1024000" readMode="2")

module(load="imudp" threads="2") # needs to be done just once
input(type="imudp" port="514" address="127.0.0.1" rcvbufSize="1m" Ruleset="udpinput")

### server model need to load and set
#module(load="imtcp" keepalive="on" keepalive.time="30" KeepAlive.Probes="1") # needs to be done just once
#input(type="imtcp" port="514" Ruleset="tcpinput")

#### GLOBAL DIRECTIVES ####

# Use default timestamp format
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Include all config files in /etc/rsyslog.d/
$IncludeConfig /etc/rsyslog.d/*.conf

# Spool files
#$WorkDirectory /var/spool/rsyslog
$WorkDirectory /var/lib/rsyslog

# Filter duplicate messages
$RepeatedMsgReduction on

$EscapeControlCharactersOnReceive off

#### RULES ####

# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.*                                                 /dev/console

# Log anything (except mail) of level info or higher.
# Don‘t log private authentication messages!
#*.info;mail.none;authpriv.none;cron.none;local0.none;local7.none                /var/log/messages
auth.info;authpriv.info;daemon.info;kern.info;syslog.info;user.info;lpr.info;mark.info                /var/log/messages

# The authpriv file has restricted access.
authpriv.*                                              /var/log/secure

# Log all the mail messages in one place.
mail.*                                                  /var/log/maillog

# Log cron stuff
cron.*                                                  /var/log/cron

# Everybody gets emergency messages
*.emerg                                                 :omusrmsg:*

# Save news errors of level crit and higher in a special file.
uucp,news.crit                                          /var/log/spooler

# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log

template(name="msg" type="string" string="%rawmsg%
")
template(name="msg" type="string" string="%rawmsg%
")

template(name="timemsg" type="string" string="%timegenerated:8:15% %msg%
")

template(name="udppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%.log")
template(name="tcppfile" type="string" string="/data/rsyslogs/%HOSTNAME%/%syslogtag%/%$year%-%$month%-%$day%_%fromhost-ip%.log")
template(name="tcpnfile" type="string" string="/data/rsyslogs/nginx/%syslogtag:F,66:1%/%$year%-%$month%-%$day%_%fromhost-ip%_%syslogtag:F,66:2%.log")

### get local code log from syslog to elasearchtich
template(name="udppname" type="string" string="%HOSTNAME%-%$year%.%$month%.%$day%")
template(name="udpelastic" type="list" option.json="on"){
    constant(value="{")
    constant(value=""PROGRAM_NAME":"") property(name="hostname")
    constant(value="","ENV":"") property(name="syslogtag")
    constant(value="","LOG_TYPE":"code_syslog",")
    constant(value=""HOSTNAME":"") property(name="$!myhostname") #get host ip
    constant(value="","TIME":"") property(name="timereported" dateFormat="rfc3339")
    constant(value="","MSG":"") property(name="msg")
    constant(value=""}")
}
### get local nginx log from syslog to elasticsearch
#ACCESS:access_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=info main;
#ERROR:error_log syslog:server=127.0.0.1:514,facility=local1,tag=m_test_2haohr_com,severity=debug;
template(name="websitename" type="string" string="%syslogtag:F,66:1%-%$year%.%$month%.%$day%")
template(name="nginxelastic" type="list" option.json="on"){
    constant(value="{")
    constant(value=""PROGRAM_NAME":"") property(name="syslogtag")
    constant(value="","ENV":"") property(name="$!env")
    constant(value="","LOG_TYPE":"") property(name="$!logtype")
    constant(value="","HOSTNAME":"") property(name="hostname") #get host ip
    constant(value="","TIME":"") property(name="timereported" dateFormat="rfc3339")
    constant(value="","MSG":"") property(name="msg")
    constant(value=""}")
}

ruleset(name="udpinput") {
    if prifilt("local0.*") then {
        action(type="omfile" dynaFile="udppfile" template="timemsg")
        action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
        set $!myhostname = "10-10-3-215";
        action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="udpelastic" dynSearchIndex="on" searchIndex="udppname" searchType="events")
    }else if $syslogfacility-text == "local1" then {
        set $!sitename = $syslogtag;
        if $syslogtag contains "_dev" then {
            set $!env = "dev";
        } else if $syslogtag contains "_test" then {
            set $!env = "test";
        } else {
            set $!env = "production";
        }
        if $syslogseverity-text == "info" then {
            set $!logtype = "nginx_access";
        } else if $syslogseverity-text == "debug" then {
            set $!logtype = "nginx_error";
        }
        action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
        action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="nginxelastic" dynSearchIndex="on" searchIndex="websitename" bulkmode="on" Action.ResumeRetryCount="-1")
    } else{
        action(type="omfile" file="var/log/test.test.log")
        stop
    }
}

template(name="filepname" type="string" string="%syslogtag:F,43:2%-%$year%.%$month%.%$day%")
template(name="fileselasticsearch" type="list" option.json="on"){
    constant(value="{")
    constant(value=""PROGRAM_NAME":"") property(name="$!programname")
    constant(value="","ENV":"") property(name="$!env")
    constant(value="","LOG_TYPE":"") property(name="$!logtype")
    constant(value="","HOSTNAME":"") property(name="hostname") #get host ip
    constant(value="","TIME":"") property(name="timereported" dateFormat="rfc3339")
    constant(value="","MSG":"") property(name="msg")
    constant(value=""}")
}
ruleset(name="fileinput"){
    if prifilt("local6.*") then {
        set $!programname = field($syslogtag,"+",2);
        if re_match($syslogtag,"G+.*+.*E") then {
            set $!logtype = "gunicorn_error";
        } else if re_match($syslogtag,"G+.*+.*A") then {
            set $!logtype = "gunicorn_access";
        } else if re_match($syslogtag,"S+.*+.*G") then {
            set $!logtype = "gunicorn_supervisor";
        } else if re_match($syslogtag,"C+.*+.*C") then {
            set $!logtype = "celery_log";
        } else if re_match($syslogtag,"S+.*+.*C") then {
            set $!logtype = "celery_supervisor";
        }

        if $syslogtag contains "D" then {
            set $!env = "dev";
        } else if $syslogtag contains "T" then {
            set $!env = "test";
        } else if $syslogtag contains "P" then {
            set $!env = "production";
        }

        action(type="omfwd" protocol="tcp" target="10.10.10.10" port="514" zipLevel="5" keepalive="on")
        action(type="omelasticsearch" server="10.10.166.166" serverport="9200" template="fileselasticsearch" dynSearchIndex="on" searchIndex="filepname" bulkmode="on" Action.ResumeRetryCount="-1")
    }
    stop
}

使用 ansible-playbook 替换所有机器

---
- hosts: GZtest
  remote_user: root
  vars:
    rsyslog_port: 514 
    protocol: udp 
  tasks:
  - name: add rsyslogd repository
    yum_repository: 
     name: rsyslog_v8
     description: Adiscon CentOS-$releasever - local packages for $basearch
     baseurl: http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch
     enabled: yes 
     gpgcheck: no  
     gpgkey: http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon 
     file: rsyslog
     protect: yes 

  - name: install rsyslog-main
    yum:
     name: rsyslog
     state: latest

  - name: install rsyslog-elastic module
    yum:
     name: rsyslog-elasticsearch
     state: latest
  - name: copy rsyslog.conf 
    copy:
     backup: yes 
     src: /etc/rsyslog.d/rsyslog_v8_client.conf
     dest: /etc/rsyslog.conf

#    shell: sed -i "s/222/{{ansible_hostname}}/g" /root/shell/test.txt
  - name: change localhost
    replace:
      dest: /etc/rsyslog.conf
      regexp: ‘10-10-3-215‘
      replace: ‘{{ansible_hostname}}‘
    notify:
    - restart rsyslogd

  handlers:
  - name: restart rsyslogd
    service:
      name: rsyslog
      enabled: yes
      state: restarted