logstash甯哥敤鎻掍欢瑙ｆ瀽
Posted
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了logstash甯哥敤鎻掍欢瑙ｆ瀽相关的知识，希望对你有一定的参考价值。
鏍囩锛?a href='http://www.mamicode.com/so/1/%e5%9c%b0%e5%9d%80' title='鍦板潃'>鍦板潃
浣嶇疆 info ems admin .mm conf https 鎿嶄綔瀹樻柟鍦板潃锛?a href="https://www.elastic.co/guide/en/logstash-versioned-plugins/current/index.html" target="_blank">https://www.elastic.co/guide/en/logstash-versioned-plugins/current/index.html
閰嶇疆鏂囦欢鍐欐硶:
# 鏃ュ織瀵煎叆input {}
# 鏃ュ織绛涢€夊尮閰嶅鐞唂ilter {}
# 鏃ュ織鍖归厤杈撳嚭output {}
鏃ュ織瑙ｆ瀽閰嶇疆鏂囦欢鐨勬鏋跺叡鍒嗕负涓変釜妯″潡,input,output,filter銆傚悗闈細涓€涓€璁茶В, 姣忎釜妯″潡閲岄潰瀛樺湪涓嶅悓鐨勬彃浠躲€?/p>input 妯″潡
鍒楀瓙1
# file涓哄父鐢ㄦ枃浠舵彃浠?鎻掍欢鍐呴€夐」寰堝,鍙牴鎹渶姹傝嚜琛屽垽鏂?br />input {
file {
path => "/var/lib/mysql/slow.log" # 瑕佸鍏ョ殑鏂囦欢鐨勪綅缃?鍙互浣跨敤*,渚嬪/var/log/nginx/*.log
Excude =>”*.gz” # 瑕佹帓闄ょ殑鏂囦欢
start_position => "beginning" # 浠庢枃浠跺紑濮嬬殑浣嶇疆寮€濮嬭,end琛ㄧず浠庣粨灏惧紑濮嬭
ignore_older => 0 # 澶氫箙涔嬪唴娌′慨鏀硅繃鐨勬枃浠朵笉璇诲彇,0涓烘棤闄愬埗,鍗曚綅涓虹
sincedb_path => "/dev/null" # 璁板綍鏂囦欢涓婃璇诲彇浣嶇疆,杈撳嚭鍒皀ull琛ㄧず姣忔閮戒粠鏂囦欢棣栬寮€濮嬭В鏋?nbsp;
type => "mysql-slow" # type瀛楁,鍙〃鏄庡鍏ョ殑鏃ュ織绫诲瀷
}
}渚嬪瓙2
# redis鎻掍欢涓哄父鐢ㄦ彃浠?鎻掍欢鍐呴€夐」寰堝,鍙牴鎹渶姹傝嚜琛屽垽鏂?nbsp;
input {
redis {
batch_count => 1 # EVAL鍛戒护杩斿洖鐨勪簨浠舵暟鐩?璁剧疆涓?琛ㄧず涓€娆¤姹傝繑鍥?鏉℃棩蹇椾俊鎭?nbsp;
data_type => "list" # logstash redis鎻掍欢宸ヤ綔鏂瑰紡
key => "logstash-test-list" # 鐩戝惉鐨勯敭鍊?nbsp;
host => "127.0.0.1" # redis鍦板潃
port => 6379 # redis绔彛鍙?nbsp;
password => "123123" # 濡傛灉鏈夊畨鍏ㄨ璇?姝ら」涓鸿璇佸瘑鐮?nbsp;
db => 0 # 濡傛灉搴旂敤浣跨敤浜嗕笉鍚岀殑鏁版嵁搴?姝や负redis鏁版嵁搴撶殑缂栧彿,榛樿涓?銆?nbsp;
threads => 1 # 鍚敤绾跨▼鏁伴噺
}
}
甯哥敤鐨?input 鎻掍欢鍏跺疄鏈夊緢澶?杩欓噷鍙妇渚嬩簡涓ょ銆傚叾浠栬繕鏈?kafka,tcp 绛夌瓑filter 妯″潡
渚嬪瓙
filter { # 鎻掍欢寰堝,杩欓噷閫夊彇鎴戜娇鐢ㄨ繃鐨勬彃浠跺仛璁茶堪
if ([message] =~ "姝ｅ垯琛ㄨ揪寮?) { drop {} } # 姝ｅ垯鍖归厤=~,!~,鍖呭惈鍒ゆ柇in,not in ,瀛楃涓插尮閰?=,!=,绛夌瓑,鍖归厤涔嬪悗鍙互鍋氫换浣曟搷浣?杩欓噷杩囨护鎺夊尮閰嶈,闄や簡鍋氳繃婊ゆ搷浣?if鍚庨潰鍙互浣滀换鎰忔搷浣?鐢氳嚦鍙互涓哄尮閰嶅埌鐨勪换鎰忚鍋氬崟鐙殑姝ｅ垯鍒嗗壊鎿嶄綔
multiline {
pattern => "姝ｅ垯琛ㄨ揪寮?
negate => true
what => "previous" # 澶氳鍚堝苟,鐢变簬涓€浜涙棩蹇楀瓨鍦ㄤ竴鏉″琛岀殑鎯呭喌,杩欎釜妯″潡鍙互杩涜鎸囧畾澶氳鍚堝苟,閫氳繃姝ｅ垯鍖归厤,鍖归厤鍒扮殑鍐呭涓婇潰鐨勫琛屽悎骞朵负涓€鏉℃棩蹇椼€?nbsp;
}
grok {
match => { "message" => "姝ｅ垯琛ㄨ揪寮? # 姝ｅ垯鍖归厤鏃ュ織,鍙互绛涢€夊垎鍓插嚭闇€瑕佽褰曠殑瀛楁鍜屽€?nbsp; }
remove_field => ["message"] # 鍒犻櫎涓嶉渶瑕佽褰曠殑瀛楁
}
date { match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"] # 璁板綍@timestamp鏃堕棿,鍙互璁剧疆鏃ュ織涓嚜瀹氱殑鏃堕棿瀛楁,濡傛灉鏃ュ織涓病鏈夋椂闂村瓧娈?涔熷彲浠ヨ嚜宸辩敓鎴?nbsp;
target=>“@timestamp” # 灏嗗尮閰嶇殑timestamp瀛楁鏀惧湪鎸囧畾鐨勫瓧娈?榛樿鏄疈timestamp }
ruby { code => "event.timestamp.time.localtime" # timestamp鏃跺尯閿佸畾 }
}output 妯″潡
渚嬪瓙1
output {
# tdout { codec => "rubydebug" } # 绛涢€夎繃婊ゅ悗鐨勫唴瀹硅緭鍑哄埌缁堢鏄剧ず
elasticsearch { # 瀵煎嚭鍒癳s,鏈€甯哥敤鐨勬彃浠?nbsp;
codec => "json" # 瀵煎嚭鏍煎紡涓簀son
hosts => ["127.0.0.1:9200"] # ES鍦板潃+绔彛
index => "logstash-slow-%{+YYYY.MM.dd}" # 瀵煎嚭鍒癷ndex鍐?鍙互浣跨敤鏃堕棿鍙橀噺
user => "admin" password => "xxxxxx" # ES濡傛灉鏈夊畨鍏ㄨ璇佸氨浣跨敤璐﹀彿瀵嗙爜楠岃瘉,鏃犲畨鍏ㄨ璇佸氨涓嶉渶瑕?nbsp;
flush_size => 500 # 榛樿500,logstash涓€娆℃€ф敀澶?00鏉＄殑鏁版嵁鍦ㄥ悜es鍙戦€?nbsp;
idle_flush_time => 1 # 榛樿1s,濡傛灉1s鍐呮病鏀掑500,杩樻槸浼氫竴娆℃€ф妸鏁版嵁鍙戠粰ES } }渚嬪瓙2
output {
redis{ # 杈撳嚭鍒皉edis鐨勬彃浠?涓嬮潰閫夐」鏍规嵁闇€姹備娇鐢?nbsp;
batch => true # 璁句负false,涓€娆push,鍙戜竴鏉℃暟鎹?true涓哄彂閫佷竴鎵?nbsp;
batch_events => 50 # 涓€娆push鍙戦€佸灏戞暟鎹?nbsp;
batch_timeout => 5 # 涓€娆push娑堣€楀灏戞椂闂?nbsp;
codec => plain # 瀵硅緭鍑烘暟鎹繘琛宑odec,閬垮厤浣跨敤logstash鐨剆eparate filter
congestion_interval => 1 # 澶氶暱鏃堕棿杩涢」涓€娆℃嫢濉炴鏌?nbsp;
congestion_threshold => 5 # 闄愬埗涓€涓猯ist涓彲浠ュ瓨鍦ㄥ灏戜釜item,褰撴暟閲忚冻澶熸椂,灏变細闃诲鐩村埌鏈夊叾浠栨秷璐硅€呮秷璐筶ist涓殑鏁版嵁
data_type => list # 浣跨敤list杩樻槸publish
db => 0 # 浣跨敤redis鐨勯偅涓暟鎹簱,榛樿涓?鍙?nbsp;
host => ["127.0.0.1:6379"] # redis 鐨勫湴鍧€鍜岀鍙?浼氳鐩栧叏灞€绔彛
key => xxx # list鎴朿hannel鐨勫悕瀛?nbsp;
password => xxx # redis鐨勫瘑鐮?榛樿涓嶄娇鐢?nbsp;
port => 6379 # 鍏ㄥ眬绔彛,榛樿6379,濡傛灉host宸叉寚瀹?鏈潯澶辨晥
reconnect_interval => 1 # 澶辫触閲嶈繛鐨勯棿闅?榛樿涓?s
timeout => 5 # 杩炴帴瓒呮椂鐨勬椂闂?nbsp;
workers => 1 # 宸ヤ綔杩涚▼
}
}
甯哥敤鎻掍欢杩樻湁寰堝,鏇村鐨勬彃浠朵娇鐢ㄥ彲浠ユ煡鐪嬪畼鏂规枃妗?br />閫氳繃涓婇潰鐨勪粙缁?鎴戜滑澶т綋鐭ラ亾浜?logstash 鐨勫鐞嗘祦绋?
input => filter => output
鎺ヤ笅鏉ュ氨鐪嬩竴瀹屾暣鐨勫簲鐢ㄤ緥瀛?br />瀹屾暣鐨勫簲鐢?
Elasticsearch slow-loginput { file { path => ["/var/log/elasticsearch/private_test_index_search_slowlog.log"] start_position => "beginning" ignore_older => 0 # sincedb_path => "/dev/null" type => "elasticsearch_slow" } } filter { grok { match => { "message" => "^/[(/d/d){1,2}-(?:0[1-9]|1[0-2])-(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])/s+(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)/]/[(TRACE|DEBUG|WARN/s|INFO/s)/]/[(?[a-z/.]+)/]/s/[(?[a-z0-9/-/.]+)/]/s/[(?[A-Za-z0-9/./_/-]+)/]/[/d+/]/s+took/[(?[/./d]+(ms|s|m))/]/,/s+took_millis/[(/d)+/]/,/s+types/[(?([A-Za-z/_]+|[A-Za-z/_]*))/]/,/s+stats/[/]/,/s+search_type/[(?[A-Z/_]+)/]/,/s+total_shards/[/d+/]/,/s+source/[(?[/s/S]+)/]/,/s+extra_source/[[/s/S]*/]/,/s*$" } remove_field => ["message"] } date { match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"] } ruby { code => "event.timestamp.time.localtime" } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-elasticsearch-slow-%{+YYYY.MM.dd}" user => "admin" password => "xxxx" } }Mysql-slow log
input { file { path => "/var/lib/mysql/slow.log" start_position => "beginning" ignore_older => 0 # sincedb_path => "/dev/null" type => "mysql-slow" } } filter { if ([message] =~ "^(//usr//local|Tcp|Time)[/s/S]*") { drop {} } multiline { pattern => "^/#/s+Time/:/s+/d+/s+(0[1-9]|[12][0-9]|3[01]|[1-9])" negate => true what => "previous" } grok { match => { "message" => "^/#/sTime/:/s+/d+/s+(?%{TIME})/n+/#/[email protected]/:/s+[A-Za-z0-9/_]+/[(?[A-Za-z0-9/_]+)/]/[email protected]/s+(?[A-Za-z0-9/_]+)/s+/[/]/n+/#/s+Query/_time/:/s+(?[0-9/.]+)/s+Lock/_time/:/s+(?[0-9/.]+)/s+Rows/_sent/:/s+(?/d+)/s+Rows/_examined/:/s+(?/d+)(/n+|/n+use/s+(?[A-Za-z0-9/_]+)/;/n+)SET/s+timestamp/=/d+/;/n+(?[/s/S]+)$" } remove_field => ["message"] } date { match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"] } ruby { code => "event.timestamp.time.localtime" } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-mysql-slow-%{+YYYY.MM.dd}" user => "admin" password => "xxxxx" } }Nginx access.log
logstash 涓唴缃?nginx 鐨勬鍒?鎴戜滑鍙绋嶄綔淇敼灏辫兘浣跨敤
灏嗕笅闈㈢殑鍐呭鍐欏叆鍒?opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/grok-patterns 鏂囦欢涓?/p>X_FOR (%{IPV4}|-)NGINXACCESS %{COMBINEDAPACHELOG} /"%{X_FOR:http_x_forwarded_for}/"ERRORDATE %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}NGINXERROR_ERROR %{ERRORDATE:timestamp}/s{1,}/[%{DATA:err_severity}/]/s{1,}(%{NUMBER:pid:int}#%{NUMBER}:/s{1,}/*%{NUMBER}|/*%{NUMBER}) %{DATA:err_message}(?:,/s{1,}client:/s{1,}(?%{IP}|%{HOSTNAME}))(?:,/s{1,}server:/s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:server_ip})?(?:, referrer:/"%{URI:referrer})?NGINXERROR_OTHER %{ERRORDATE:timestamp}/s{1,}/[%{DATA:err_severity}/]/s{1,}%{GREEDYDATA:err_message}涔嬪悗鐨?log 閰嶇疆鏂囦欢濡備笅
input { file { path => [ "/var/log/nginx/www-access.log" ] start_position => "beginning" # sincedb_path => "/dev/null" type => "nginx_access" } } filter { grok { match => { "message" => "%{NGINXACCESS}"} } mutate { convert => [ "response","integer" ] convert => [ "bytes","integer" ] } date { match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"] } ruby { code => "event.timestamp.time.localtime" } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-nginx-access-%{+YYYY.MM.dd}" user => "admin" password => "xxxx" } }Nginx error.log
input { file { path => [ "/var/log/nginx/www-error.log" ] start_position => "beginning" # sincedb_path => "/dev/null" type => "nginx_error" } } filter { grok { match => [ "message","%{NGINXERROR_ERROR}", "message","%{NGINXERROR_OTHER}" ] } ruby { code => "event.timestamp.time.localtime" } date { match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss"] } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-nginx-error-%{+YYYY.MM.dd}" user => "admin" password => "xxxx" } }php error.log
input { file { path => ["/var/log/php/error.log"] start_position => "beginning" # sincedb_path => "/dev/null" type => "php-fpm_error" } } filter { multiline { pattern => "^/[(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}[/s/S]+" negate => true what => "previous" } grok { match => { "message" => "^/[(?(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}/s+%{TIME}?)/s+[A-Za-z]+//[A-Za-z]+/]/s+(?(?:[A-Z]{3}/s+[A-Z]{1}[a-z]{5,7}|[A-Z]{3}/s+[A-Z]{1}[a-z/s]{9,11}))/:/s+(?[/s/S]+$)" } remove_field => ["message"] } date { match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"] } ruby { code => "event.timestamp.time.localtime" } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-php-error-%{+YYYY.MM.dd}" user => "admin" password => "xxxxx" } }Php-fpm slow-log
input { file { path => ["/var/log/php-fpm/www.slow.log"] start_position => "beginning" # sincedb_path => "/dev/null" type => "php-fpm_slow" } } filter { multiline { pattern => "^$" negate => true what => "previous" } grok { match => { "message" => "^/[(?(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}/s+%{TIME})/]/s+/[[a-z]{4}/s+(?[A-Za-z0-9]{1,8})/]/s+[a-z]{3}/s+(?/d{1,7})/n(?[/s/S]+$)" } remove_field => ["message"] } date { match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"] } ruby { code => "event.timestamp.time.localtime" } } output { elasticsearch { codec => "json" hosts => ["127.0.0.1:9200"] index => "logstash-php-fpm-slow-%{+YYYY.MM.dd}" user => "admin" password => "xxxx" } }log 瑙ｆ瀽閰嶇疆鏂囦欢缁熶竴鏀惧湪/etc/logstash/conf.d 鐩綍涓?涓嶈繃涔熷彲浠ヤ换鎰忔斁缃?缁熶竴璧锋潵鏈€濂姐€?br />鍦ㄥ涓厤缃枃浠剁殑鏃跺€?涓嶈兘浣跨敤濡備笅鍛戒护杩愯logstash:
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/(鎴栬€呮湁涓?)
杩欎釜鍛戒护浼氭嫾鎺ラ厤缃枃浠?涓嶄細鍗曚釜浣跨敤,浼氭姤閿欍€?br />濡傛灉鏈夊涓厤缃枃浠?灏变竴涓竴涓惎鍔?
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_error.conf &
浣嗘槸杩欐牱涔熷緢楹荤儲,濡傛灉閰嶇疆鏂囦欢寰堝鐨勬儏鍐典笅闇€瑕佷竴涓釜鏉?骞朵笖鍚姩
閫熷害杩樺緢鎱?鍐欎簡涓€涓祴璇曡剼鏈敤鏉ユ柟渚夸娇鐢?浠呬緵鍙傝€?#!/bin/bash # /閰嶇疆鏂囦欢瀛樻斁鐩綍鏍规嵁闇€姹傝嚜宸辨洿鏀? conf_path=/etc/logstash/conf.d conf_name=$( ls ${conf_path} ) case $1 in start) echo "-----------please wait.----------" echo "The start-up process is too slow." for cf in ${conf_name} do /opt/logstash/bin/logstash -f $conf_path/$cf > /dev/null 2>&;1 &; if [ $? -ne 0 ];then echo 鈥楾he 鈥?{cf}鈥?start-up failed.鈥? fi sleep 20 done echo "start-up success." ;; stop) ps -ef |grep logstash |grep -v grep > /dev/null 2>&;1 if [ $? -eq 0 ];then ps -ef|grep logstash |grep -v grep |awk 鈥榹print $2}鈥榺xargs kill -9 > /dev/null 2>&;1 sleep 2 echo "Stop success." fi ;; restart) ps -ef |grep logstash |grep -v grep 2>&;1 if [ $? -eq 0 ];then ps -ef|grep logstash |grep -v grep |awk 鈥榹print $2}鈥榺xargs kill -9 > /dev/null 2>&;1 sleep 3 echo "Stop success." fi echo "-----------please wait.----------" echo "The start-up process is too slow." for cf in ${conf_name} do /opt/logstash/bin/logstash -f $conf_path/$cf > /dev/null 2>&;1 &; if [ $? -ne 0 ];then echo 鈥楾he 鈥?{cf}鈥?start-up failed.鈥? fi sleep 10 done echo "start-up success." ;; *) echo "Usage: "$0" {start|stop|restart|}" exit 1; ;; esac鑴氭湰鐨勫悕瀛椾腑涓嶈鍖呭惈 logstash,杩欓噷淇濆瓨涓?log_stash.sh锛屼娇鐢?/log_stash.sh (start|stop|restart) 鏉ユ墽琛岃剼鏈€?br />
以上是关于logstash甯哥敤鎻掍欢瑙ｆ瀽的主要内容，如果未能解决你的问题，请参考以下文章
33 涓彁楂樺墠绔伐浣滄晥鐜囩殑 VSCode 瀹炵敤鎻掍欢
闆跺熀纭€瀛ginx銆?銆憒 Nginx 甯哥敤鐨勫懡浠ゅ拰閰嶇疆鏂囦欢