filebeat+nginx 绘图时url不能模糊搜索的问题

Posted 2020-11-26 纪录

filebeat+nginx 绘图时url不能模糊搜索的问题

　　1.修改之前nginx 日志配置为

　　这里$request_time和$upstream_response_time打上引号是因为，如果取不到这个值，这个值就位 - ，logstash会报错，所以就由字符串再转为float类型

log_format json ‘{"time": "$time_iso8601", ‘
                    ‘"remote_addr": "$remote_addr", ‘
                    ‘"transactionId": "$http_TraceId", ‘
                    ‘"referer": "$http_referer", ‘
                    ‘"website": "$http_host", ‘
                    ‘"uri": "$uri", ‘
                    ‘"status": $status, ‘
                    ‘"bytes": $body_bytes_sent, ‘
                    ‘"agent": "$http_user_agent", ‘
                    ‘"x_forwarded": "$http_x_forwarded_for", ‘
                    ‘"up_addr": "$upstream_addr", ‘
                    ‘"up_host": "$upstream_http_host", ‘
                    ‘"upstreamresponsetime": "$upstream_response_time", ‘
                    ‘"responsetime": "$request_time"‘
                    ‘}‘;

 　　logstash配置

[root@prod-logstash02-E logstash]# cat conf.d/nginx-output-kafka.conf 
input {
       kafka {
         bootstrap_servers => "172.27.27.220:9092,172.27.27.221:9092,172.27.27.222:9092"
         topics => ["nginxlogs"]
         codec => "json"
	 type => "nginx-access"
       }
}

filter {
	if [type] == "nginx-access" {
		json {
			source => "message"
		}
	}

	mutate{
        convert => ["responsetime","float"]
        convert => ["upstreamresponsetime","float"]
        }

	geoip {
	source => "remote_addr"
	#database => "/opt/logstash-6.5.4/Geoip/GeoLite2-City_20191022/GeoLite2-City.mmdb"
	}
}


output {    
	if [fields][type] =~ "nginxlogs" {
	elasticsearch {
	  hosts => [ "172.27.27.220:9200","172.27.27.221:9200","172.27.27.222:9200" ]
	  index => "logstash-nginxaccess-%{+YYYY.MM.dd}"
	  template_overwrite => true
	  user => "xxx"
	  password => "xxxx"
	  #document_type => "%{[type]}"
	}
	}
	#stdout { codec => rubydebug }

}

　　查看uri的mapping类型为keyword，在绘图时不能模糊搜索

2、修改之后的配置

　　nginx.conf

    log_format json ‘{"time": "$time_iso8601", ‘
                    ‘"remote_addr": "$remote_addr", ‘
                    ‘"transactionId": "$http_TraceId", ‘
                    ‘"referer": "$http_referer", ‘
                    ‘"website": "$http_host", ‘
                    ‘"request": "$request", ‘
                    ‘"status": $status, ‘
                    ‘"bytes": $body_bytes_sent, ‘
                    ‘"agent": "$http_user_agent", ‘
                    ‘"x_forwarded": "$http_x_forwarded_for", ‘
                    ‘"up_addr": "$upstream_addr", ‘
                    ‘"up_host": "$upstream_http_host", ‘
                    ‘"upstreamresponsetime": "$upstream_response_time", ‘
                    ‘"responsetime": "$request_time"‘
                    ‘}‘;

 　　logstash配置

[root@prod-logstash02-E logstash]# cat conf.d/nginx-output-kafka.conf 
input {
       kafka {
         bootstrap_servers => "172.27.27.220:9092,172.27.27.221:9092,172.27.27.222:9092"
         topics => ["nginxlogs"]
         codec => "json"
	 type => "nginx-access"
       }
}

filter {
	if [type] == "nginx-access" {
		json {
			source => "message"
		}
	}

	mutate{
        convert => ["responsetime","float"]
        convert => ["upstreamresponsetime","float"]
        }
	
	mutate {
		add_field => { "request1" => "%{request}" }
	}

	grok {
		match => {"request1" => "(?:%{WORD:requestmethod} %{URIPATH:url}(?:%{URIPARAM:params})?(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})"}
		remove_field => ["request1"]
	}

	geoip {
		source => "remote_addr"
		#database => "/opt/logstash-6.5.4/Geoip/GeoLite2-City_20191022/GeoLite2-City.mmdb"
	}
}


output {    
	if [fields][type] =~ "nginxlogs" {
	elasticsearch {
	  hosts => [ "172.27.27.220:9200","172.27.27.221:9200","172.27.27.222:9200" ]
	  index => "logstash-nginxaccess-%{+YYYY.MM.dd}"
	  template_overwrite => true
	  user => "xxx"
	  password => "xxxx"
	  #document_type => "%{[type]}"
	}
	}
	#stdout { codec => rubydebug }

}

 

　　这样配置后，url的类型为text，这时候才可以模糊搜索。