logstash.conf示例

Posted 2022-11-28 PoetryAndTheDistance

参考logstash的一个入门资料： http://doc.yonyoucloud.com/doc/logstash-best-practice-cn/index.html

输出ES时创建的索引模板定义：https://www.cnblogs.com/you-you-111/p/9844131.html

https://www.cnblogs.com/cangqinglang/p/12187801.html

 

ongdb_query_log.conf

#输入
input 
  	beats 
		port  => "5044"
		client_inactivity_timeout => 36000
 	


#过滤
filter 
	grok 
		match => 
			"message" =>[
				"%TIMESTAMP_ISO8601:log_timestamp %LOGLEVEL:log_level  %INT:time_consuming %USERNAME:time_consuming_unit:.*client/%IP:client_ip:%INT:client_port.*%IP:server_ip:%INT:server_port.* -[ |\\r\\n]%GREEDYDATA:cypher - .* - .*",
				"%TIMESTAMP_ISO8601:log_timestamp %LOGLEVEL:log_level  %INT:time_consuming %USERNAME:time_consuming_unit:.*(ongdb|graph-user1|graph-user2|neo4j|graph-user3|techfin|esg) -[ |\\r\\n]%GREEDYDATA:cypher - ",
				"%TIMESTAMP_ISO8601:log_timestamp %LOGLEVEL:log_level  %INT:time_consuming %USERNAME:time_consuming_unit:.* -[ |\\r\\n]%GREEDYDATA:cypher - .* - .*"
			]
		
#		add_field => ["day", "%+YYYY.MM.dd"]
#		add_field => ["received_at", "%@timestamp"]
#		add_field => ["received_log", "%host"]
#		remove_field => ["host"]
#		add_field => ["received_logstash", "%host"]
#		remove_field => ["message","@timestamp","tags","log","input","agent","ecs"]

		
		add_field => ["received_at", "%@timestamp"]
		add_field => ["received_from", "%host"]
		add_field => ["day", "%+YYYY.MM.dd"]
		remove_field => ["message","@timestamp","tags","log","input","agent","ecs","host"]
	
#	mutate 
#        		convert => ["time_consuming", "int"]
#   	
#	date 
#		match => [ "log_timestamp", "YYYY-MMM-dd HH:mm:ss.SSS Z" ]
#	

#输出
output 
	elasticsearch 
#        		hosts => "http://10.20.13.130:9200"
        		hosts => "http://10.20.8.155:9200"
		index => "logstash_ongdb_querylog_%day"
#		index => "ongdb_querylog"
		template => "/home/ubuntu/ongdbETL/logstash-7.5.1/bin/conf/logstash_ongdb_querylog.json"
            		template_name => "logstash_ongdb_querylog_*"
           		template_overwrite => true
    	
	stdout 

 filebeat.yml

#运行命令	./filebeat -c filebeat_neo4j_log.yml -e
filebeat.inputs:
- type: log
  enabled: true
  encoding: utf-8
  paths:
    - /home/ongdb/ongdb-enterprise-3.5.22/logs/query.*
  multiline.pattern: '^\\d4-\\d2-\\d2.*'
  multiline.negate: true
  multiline.match: after

output.logstash:
  hosts: ["10.20.4.28:5044"]

logstash_neo4j_querylog.json 

  "template": "logstash_ongdb_querylog_*",
  "order": 1,
  "settings": 
    "number_of_replicas": 0,
    "number_of_shards": 1,
    "refresh_interval": "60s",
    "translog": 
      "flush_threshold_size": "256mb"
    ,
    "merge": 
      "scheduler": 
        "max_thread_count": "1"
      
    ,
    "index": 
      "routing": 
        "allocation": 
          "total_shards_per_node": "1"
        
      
    ,
    "analysis": 
      "normalizer": 
        "my_normalizer": 
          "type": "custom",
          "filter": [
            "lowercase",
            "asciifolding"
          ]
        
      
    
  ,
  "mappings": 
    "properties": 
      "time_consuming": 
        "index": true,
        "store": true,
        "type": "integer"
      ,
      "time_consuming_unit": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "client_ip": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "client_port": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "server_ip": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "server_port": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "cypher": 
        "index": true,
        "store": true,
        "type": "text"
      ,
      "received_from": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "received_at": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "log_level": 
        "index": true,
        "store": true,
        "type": "keyword"
      ,
      "log_timestamp": 
        "index": true,
        "store": true,
        "type": "keyword"
      
    
  ,
        "aliases": 
            "logstash_neo4j_querylog":