ELK日志平台----解耦配置文件

Posted 2020-09-27

本文记录了三个配置文件：

第一个：all.conf 通过一个配置文件，配置输入输出，实例；

第二个：shipper.conf配置logstash收集日志内容到redis里；

第三个：indexer.conf配置logstash从redis里读取日志内容输出到Elasticsearch里。

第二个跟第三个配置解耦收集日志

ELK 解耦

logstash ---------->redis ---------->logstash -------->elasticsearch----------->kibana

收集                     消息队列                 分析                       存储                                 显示

192.168.1.87       192.168.1.87         192.168.1.88        192.168.1.87                192.168.1.87

all.conf 通过一个配置文件，配置输入输出，实例；
[[email protected] ~]# cat all.conf
input {
    file {
       	path => "/var/log/messages"
    	type => "system"	
    	start_position => "beginning"
    	}
    file {
        path => "/var/log/nginx/access_json.log"
        codec => "json"
        type => "nginx_log"
    	start_position => "beginning"
    }
    file {
        path => "/var/log/elasticsearch/Mint.log"
        type => "es-error"
        start_position => "beginning"
        codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
        }
    }
    syslog {
        type => "system-syslog"
        host => "192.168.1.87"
        port => "514"
    }
}
output {
    if [type] == "system" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "es-error" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "es-error-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx_log" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "nginx_log-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "system-syslog" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

shipper.conf配置logstash收集日志内容到redis里；
[[email protected] ~]# cat shipper.conf
input {
    file {
   	path => "/var/log/messages"
	type => "system"	
	start_position => "beginning"
	}
    file {
        path => "/var/log/nginx/access_json.log"
        codec => "json"
        type => "nginx_log"
	start_position => "beginning"
}
    file {
        path => "/var/log/elasticsearch/Mint.log"
        type => "es-error"
	start_position => "beginning"
        codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
        }
    }
    
    syslog {
        type => "system-syslog"
        host => "192.168.1.87"
        port => "514"
        }
}
output {
    if [type] == "system" {
        redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
        }
    }
    if [type] == "es-error" {
        redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
        }
    }
    if [type] == "nginx_log" {
    redis {
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx_log"
       }
    }
    if [type] == "system-syslog" {
        redis {
            host => "192.168.1.87 "
            port => "6379"
            db => "6"
            data_type => "list"
            key => "system-syslog"
        }
    }
}

indexer.conf 配置logstash从redis里读取日志内容输出到Elasticsearch里。
[[email protected] ~]# cat indexer.conf
input {
    redis {
        type => "system"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
    }
    redis {
        type => "es-error"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
    }
    redis {
        type => "nginx_log"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx_log"
    }
    redis {
        type => "system-syslog"
        host => "192.168.1.87 "
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system-syslog"
    }
}
output {
    if [type] == "system" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "es-error" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "es-error-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx_log" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "nginx_log-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "system-syslog" {
        elasticsearch {
            hosts => ["192.168.1.87:9200"]
            index => "system-syslog-%{+YYYY.MM.dd}"
        }
    }
}

本文出自 “share,open source” 博客，请务必保留此出处http://liqilong2010.blog.51cto.com/3029053/1946598