filebeat采集nginx日志，业务日志，阿里云sms，slb日志

Posted 2022-09-09 Sicc1107

filebeat采集nginx日志，业务日志，阿里云sms，slb日志

nginx日志

1 设置nginx日志为json模式

编辑nginx.conf-> log_format main

user  root;
worker_processes  4;
worker_rlimit_nofile 65535;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events 
    use epoll;
    worker_connections 65535;
    multi_accept on;



http 
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    fastcgi_intercept_errors on;
log_format main   '"@timestamp":"$time_iso8601",'
                        '"@source":"$server_addr",'
                        '"hostname":"$hostname",'
                        '"ip":"$http_x_forwarded_for",'
                        '"client":"$remote_addr",'
                        '"request_method":"$request_method",'
                        '"scheme":"$scheme",'
                        '"domain":"$server_name",'
                        '"referer":"$http_referer",'
                        '"request":"$request_uri",'
                        '"args":"$args",'
                        '"size":$body_bytes_sent,'
                        '"status": $status,'
                        '"responsetime":$request_time,'
                        '"upstreamtime":"$upstream_response_time",'
                        '"upstreamaddr":"$upstream_addr",'
                        '"http_user_agent":"$http_user_agent",'
                        '"https":"$https"'
                        '';


    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    tcp_nopush on;
	tcp_nodelay on;
	keepalive_timeout 600000000;
	client_body_timeout 600000000;
	send_timeout 600000000;
	types_hash_max_size 2048;
	server_tokens off;

	server_names_hash_max_size 4096;
	server_names_hash_bucket_size 128;
	server_name_in_redirect off;

	client_header_buffer_size 128k;
	large_client_header_buffers 4 256k;
	client_max_body_size 10240m;
	
	client_body_buffer_size 1024k;
	proxy_buffer_size 512k;
	proxy_buffers 8 512k;
	proxy_busy_buffers_size 512k;
	proxy_temp_file_write_size 512k;
	proxy_connect_timeout  750000s;
	proxy_http_version 1.1;
	proxy_set_header Connection "";
	proxy_send_timeout  750000s;
	proxy_read_timeout  750000s;

	gzip on;

	# gzip_vary on;
	gzip_proxied any;
	gzip_min_length 1000;
	gzip_comp_level 6;
	gzip_buffers 16 8k;
	gzip_http_version 1.1;
	gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
	gzip_disable "MSIE [1-6].(?!.*SV1)";
        gzip_vary on;
limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s;
    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;

2列举一条nginx日志信息

"@timestamp":"2022-09-08T00:42:19+00:00","@source":"172.16.1.2","hostname":"thz","ip":"-","client":"60.190.227.50","request_method":"POST","scheme":"https","domain":"tapi.shining3d.com","referer":"-","request":"/da/ed/status","args":"-","size":1552,"status": 200,"responsetime":0.069,"upstreamtime":"0.052","upstreamaddr":"172.16.1.1:3000","http_user_agent":"Go-http-client/1.1","https":"on"
"@timestamp":"2022-09-08T00:42:19+00:00","@source":"172.16.1.2","hostname":"thz","ip":"-","client":"10.10.10.50","request_method":"POST","scheme":"https","domain":"tapi.shining3d.com","referer":"-","request":"/da/ed/status","args":"-","size":1552,"status": 200,"responsetime":0.063,"upstreamtime":"0.052","upstreamaddr":"172.16.1.2:3000","http_user_agent":"Go-http-client/1.1","https":"on"

3 接口业务日志列举

"level":"debug","time":"2022-09-08T00:45:44Z","caller":"/var/jenkins_home/workspace/test-app1/vendor/git.aaaa.com/cloud/util/app1/sync.go:76","message":"TriggerSyncSQL:PublishMessage","params":"syncSQLMsg":"id":"","actual_tb_name":"order_attachs","sync_sql":"UPDATE order_attachs SET current_format = ? WHERE id = ? AND order_id = ? AND current_format = ?","sql_args":["plylor,stliew","921944a","e0828","ply_color,stl,preview3d,preview"],"row_id":"e828","create_on":"2022-09-08 00:45:44.138088","is_strict":true,"current_node":"hz","nodes":["hz","hz"],"useMs":17

"level":"debug","time":"2022-09-08T00:45:44Z","caller":"/var/jenkins_home/workspace/test-app1/vendor/git.aaaa.com/cloud/util/appp1/sync.go:76","message":"TriggerSyncSQL:PublishMessage","params":"syncSQLMsg":"id":"","actual_tb_name":"","sync_sql":"UPDATE 3d_dder_attachs SET status = ?, default_status = ?, err = ? WHERE order_id = ? and dfs_id = ? and attach_type = ?","sql_args":["error","error","null","e07d8351-a6f6-528","den","full"],"row_id":"-af828","create_on":"2022-09-08 00:4.148446","is_strict":true,"current_node":"pro","nodes":["hz","hz"],"useMs":11

4 前端日志列举

"service":"Cloud2.0","logger":"application-logger","hostname":"145b","level":"info","msg":"Use intranet node! https://www.baidu.com/","meta":"service":"version":"1.0.0","node_env":"production","logger":"time":"2022-09-08T00:48:14.629Z","event":
"service":"Cloud2.0","logger":"application-logger","hostname":"145b","level":"info","msg":"Use intranet node! https://www.baidu.com/","meta":"service":"version":"1.0.0","node_env":"production","logger":"time":"2022-09-08T00:49:14.647Z","event":
"service":"Cloud2.0","logger":"application-logger","hostname":"145b","level":"info","msg":"Use intranet node! https://www.baidu.com/","meta":"service":"version":"1.0.0","node_env":"production","logger":"time":"2022-09-08T00:49:15.035Z","event":

5 列举阿里云sms slb 这两个是通过logstash接入到本地文件的日志，看之前的文章 阿里云日志消费

#短信日志
"receiver":"56933","templateId":"2286","delay":"12000","vendor":"9","@timestamp":"2022-09-07T20:54:59.678Z","bill_type":"8","content":"Verification code 69368, welcome to be a new user, thank you!","out_id":"null","signName":"[Shining3D]","@version":"1","__time__":"188888888888","mc_code":"DELIVERED","sms_size":"1","status":"4","type":"sms","__topic__":"sms_access_log","__source__":"log_service","code":"DELIVERED"

#slb日志
"vip_addr":"172.0.0.7","http_x_forwarded_for":"10.9.6.17, 10.9.6.17","@timestamp":"2022-09-08T00:54:49.645Z","http_referer":"-","http_x_real_ip":"-","slb_vport":"80","host":"hzapi.shining3ddata.com","upstream_response_time":"0.018","request_method":"GET","time":"2022-09-08T08:54:46+08:00","tcpinfo_rtt":"1506","ssl_cipher":"-","write_response_time":"0","status":"200","request_length":"261","type":"slb","__source__":"log_service","scheme":"http","http_user_agent":"axios/0.26.1","slbid":"lb-bp1v06g83y05kex2n3acq","request_time":"0.019","upstream_status":"200","read_request_time":"0","client_ip":"172.1.1.3","body_bytes_sent":"34","upstream_addr":"172.1.1.3:80","http_host":"hzapi.com","__time__":"18888888888","@version":"1","client_port":"54630","server_protocol":"HTTP/1.1","request_uri":"/ipc","__topic__":"slb_layer7_access_log","ssl_protocol":"-"

6 filebeat配置文件

# Needed for Graylog
fields_under_root: true
fields.collector_node_id: $sidecar.nodeName
fields.gl2_source_collector: $sidecar.nodeId

#nginx日志
filebeat.inputs:
- input_type: log
  paths:
    - /data/logs/nginx/*.log
  exclude_lines: ['password','secretKey','token','Token','accessKey','secretKey','clientSecret','appID']
  type: nginx-log  
  processors:
    - decode_json_fields:
        fields: ["message"]
        process_array: false
        max_depth: 1
        target: ""
        overwrite_keys: false


#接口日志 用 dissect 进行分割
- input_type: log
  paths:
    - /data/logs/*/service.log-*
  exclude_lines: ['password','secretKey','token','Token','accessKey','secretKey','clientSecret','appID']
  type: backend-log  
  processors:
    - dissect:
        tokenizer: "\\"level\\":%@level,\\"time\\":%@time,\\"caller\\":%@caller,\\"message\\":%@message,\\"params\\":%@params"
        field: "message"


#前端日志 用 dissect 进行分割
- input_type: log
  paths:
    - /data/hzit/logs/*/error.log
    - /data/hzit/logs/*/all.log
  exclude_lines: ['password','secretKey','token','Token','accessKey','secretKey','clientSecret','appID']
  type: front-log  
  processors:
    - dissect:
        tokenizer: "\\"service\\":%@service,\\"logger\\":%@logger,\\"hostname\\":%@hostname,\\"level\\":%@level,\\"msg\\":%@msg,\\"meta\\":%@meta"
        field: "message"


##slb，sms日志本身josn格式
- input_type: log
  paths:
    - /data/hzit/logs/sms/*.log
    - /data/hzit/logs/slb/*.log
  exclude_lines: ['password','secretKey','token','Token','accessKey','secretKey','clientSecret','appID']
  type: sms-slb-log  
  processors:
    - decode_json_fields:
        fields: ["message"]
        process_array: false
        max_depth: 1
        target: ""
        overwrite_keys: false

filter:
 - mutate:
      convert: [ "status","integer","size","integer","upstreatime","float"]
      remove_field: ["message"]
 - geoip:
      source: ["filebeat_ip"]


output.logstash:
   hosts: ["172.1.1.3:5000"]
path:
  data: /var/lib/graylog-sidecar/collectors/filebeat/data
  logs: /var/lib/graylog-sidecar/collectors/filebeat/log