ELK专题使用redis与logstash结合收集数据

Posted 2020-12-21

ELK--使用redis与logstash结合收集数据

kibana

kibana

是为elasticsearch提供web可视化界面，为用户提供数据展示

kibana安装

[root@kibana ~]# ls

1. anaconda-ks.cfg GeoLite2-City.tar.gz kibana-7.1.1-x86_64.rpm

2. [root@kibana ~]# yum -y install kibana-7.1.1-x86_64.rpm

修改kibana配置文件

#开启端口

#设置侦听端口

#设置elasticsearch主机#设置语言，不用汉化

[root@kibana ~]# cat /etc/kibana/kibana.yml | grep -v "#" | grep -v "^$"
server.port: 5601

server.host: "192.168.122.10"

elasticsearch.hosts: ["http://192.168.122.20:9200","http://192.168.122.30:9200"]
i18n.locale: "zh-CN"

启动服务

1 # systemctl start kibana;systemctl enable kibana;systemctl status kibana

访问测试页面

URL：http://kibana_ip:5601

使用?lebeat收集数据

• filebeat.inputs:
  
  >   - type: log enabled: true paths:
  
  - /var/log/nginx/access.log filebeat.config.modules:
  
  >   path: ${path.config}/modules.d/*.yml reload.enabled: false
  
  >   setup.template.settings: index.number_of_shards: 1
  
  setup.kibana: output.logstash:
  
  >   hosts: ["192.168.122.40:5044"]
  
  processors:
  
  -   add_host_metadata: ~
  
  -   add_cloud_metadata: ~

#cat /etc/filebeat/filebeat.yml

# cat /etc/logstash/conf.d/remote_filebeat_nginx.conf input {

>   beats {

>   port => 5044

>   host => "0.0.0.0"

>   }

}

output {

>   elasticsearch {

>   hosts => ["192.168.122.20:9200","192.168.122.30:9200"]

>   index => "remote_filebeat_nginx_app-%{+YYYY.MM.dd}"

>   }

>   stdout {

>   codec => rubydebug

>   }

}

使用redis与logstash结合收集数据

#logstash to redis

#在需要收集nginx日志服务

# cat /etc/logstash/conf.d/logstash_to_redis.conf input {

>   file {

path => "/var/log/nginx/access.log"

start_position => "beginning"

}

}

filter {

}

output {

>   redis {

>   host => ["192.168.122.40:6379"]

>   password => "123456"

>   db => "0"

>   data_type => "list"

>   key => "logstashtoredis"

>   }

}

requirepass 123456 #480 line

#启动服务

#systemctl enable redis #systemctl start redis

#验证

#redis-cli -h 192.168.122.40 -a 123456

>keys *

#61 line

bind 0.0.0.0

#cat /etc/redis.conf

#在redis服务器上配置

#配置logstash连接es

# cat /etc/logstash/conf.d/logstash_from_redis.conf input {

redis {

host => "192.168.122.40"

port => 6379

password => "123456"

db => "0"

data_type => "list"

key => "logstashtoredis"

}

filter {

}

output {

elasticsearch {

hosts => ["http://192.168.122.20:9200","http://192.168.122.30:9200"]

index => "logstashtoredis-redisfromlogstash-%{+YYYY.MM.dd}"

}

stdout { codec=> rubydebug }

}

使用?lebeat与redis结合收集数据

1 #在web服务器上使用filebeat收集日志 2

3 ## cat /etc/filebeat/filebeat.yml

4 ###################### Filebeat Configuration
Example ######################### 5

1. # This file is an example configuration file highlighting only the most
   common

2. # options. The filebeat.reference.yml file from the same directory contains
   all the

3. # supported options with more comments. You can use it as a reference. 9 #

4. # You can find the full configuration reference here:

5. #
   https://www.elastic.co/guide/en/beats/filebeat/index.html
   12

6. # For more available modules and options, please see the
   filebeat.reference.yml sample

7. # configuration file. 15

   16 #========================== Filebeat inputs

17

18 filebeat.inputs:

19

8. # Each - is an input. Most options can be set at the input level, so

9. # you can use different inputs for various configurations.

10. # Below are the input specific configurations. 23

    24 - type: log 25

11. # Change to true to enable this input configuration.

12. enabled: true 28

13. # Paths that should be crawled and fetched. Glob based paths.

14. paths:

15. - /var/log/nginx/access.log

16. #- c:programdataelasticsearchlogs* 33

17. # Exclude lines. A list of regular expressions to match. It drops the lines
    that are

18. # matching any regular expression from the list.

19. #exclude_lines: [‘^DBG‘] 37

20. # Include lines. A list of regular expressions to match. It exports the
    lines that are

21. # matching any regular expression from the list.

22. #include_lines: [‘^ERR‘, ‘^WARN‘] 41

23. # Exclude files. A list of regular expressions to match. Filebeat drops the
    files that

24. # are matching any regular expression from the list. By default, no files
    are dropped.

25. #exclude_files: [‘.gz$‘] 45

26. # Optional additional fields. These fields can be freely picked

27. # to add additional information to the crawled log files for filtering

28. #fields:

29. # level: debug

30. # review: 1 51

    52 ### Multiline options 53

31. # Multiline can be used for log messages spanning multiple lines. This is
    common

32. # for Java Stack Traces or C-Line Continuation 56

33. # The regexp Pattern that has to be matched. The example pattern matches
    all lines starting with [

34. #multiline.pattern: ^[ 59

35. # Defines if the pattern set under pattern should be negated or not.
    Default is false.

36. #multiline.negate: false 62

37. # Match can be set to "after" or "before". It is used to define if lines
    should be append to a pattern

38. # that was (not) matched before or after or as long as a pattern is not
    matched based on negate.

39. # Note: After is the equivalent to previous and before is the equivalent to
    to next in Logstash

40. #multiline.match: after 67

    68

    69 #============================= Filebeat modules

70

41. filebeat.config.modules:

42. # Glob pattern for configuration loading

43. path: ${path.config}/modules.d/*.yml

74

1. # Set to true to enable config reloading

2. reload.enabled: false 77

3. # Period on which files under path should be checked for changes

4. #reload.period: 10s 80

   81 #==================== Elasticsearch template setting
   ========================== 82

5. setup.template.settings:

6. index.number_of_shards: 1

7. #index.codec: best_compression

8. #_source.enabled: false 87

   88 #================================ General

89

9. # The name of the shipper that publishes the network data. It can be used
   to group

10. # all the transactions sent by a single shipper in the web interface.

11. #name:

    93

12. # The tags of the shipper are included in their own field with each

13. # transaction published.

14. #tags: ["service-X", "web-tier"] 97

15. # Optional fields that you can specify to add additional information to the

16. # output.

17. #fields:

18. # env: staging 102

    103

    104 #============================== Dashboards

19. # These settings control loading the sample dashboards to the Kibana index.
    Loading

20. # the dashboards is disabled by default and can be enabled either by
    setting the

21. # options here or by using the `setup` command.

22. #setup.dashboards.enabled: false 109

23. # The URL from where to download the dashboards archive. By default this
    URL

24. # has a value which is computed based on the Beat name and version. For
    released

25. # versions, this URL points to the dashboard archive on the
    artifacts.elastic.co

26. # website.

27. #setup.dashboards.url:

28. 116 #============================== Kibana

117

29. # Starting with Beats version 6.0.0, the dashboards are loaded via the
    Kibana API.

30. # This requires a Kibana endpoint configuration.

31. setup.kibana:

32. # Kibana Host

33. # Scheme and port can be left out and will be set to the default (http and
    5601)

34. # In case you specify and additional path, the scheme is required:
    http://localhost:5601/path

35. # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601

36. #host: "localhost:5601" 127

37. # Kibana Space ID

38. # ID of the Kibana Space into which the dashboards should be loaded. By
    default,

39. # the Default Space will be used.

40. #space.id:

41. 133 #============================= Elastic Cloud

134

135 # These settings simplify using filebeat with the Elastic Cloud
(https://cloud.elastic.co/).

136

42. # The cloud.id setting overwrites the `output.elasticsearch.hosts` and

43. # `setup.kibana.host` options.

44. # You can find the `cloud.id` in the Elastic Cloud web UI.

45. #cloud.id:

46. # The cloud.auth setting overwrites the `output.elasticsearch.username`
    and

47. # `output.elasticsearch.password` settings. The format is
    `<user>:<pass>`.

48. #cloud.auth:

49. 146 #================================ Outputs

147

148 # Configure what output to use when sending the data collected by the
beat. 149

150 #-------------------------- Elasticsearch output

50. #output.elasticsearch:

51. # Array of hosts to connect to.

52. #hosts: ["localhost:9200"] 154

53. # Optional protocol and basic auth credentials.

54. #protocol: "https"

55. #username: "elastic"

56. #password: "changeme" 159

    160 #----------------------------- Logstash output

57. #output.logstash:

58. # The Logstash hosts

    163 #hosts: ["192.168.122.40:5044"]

    164

59. # Optional SSL. By default is off.

60. # List of root certificates for HTTPS server verifications

61. #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] 168

62. # Certificate for SSL client authentication

63. #ssl.certificate: "/etc/pki/client/cert.pem" 171

64. # Client Certificate Key

65. #ssl.key: "/etc/pki/client/cert.key" 174

    175

    176

    177

178 #----------------------------- redis output

1. output.redis:

2. # The redis hosts

   181 hosts: ["192.168.122.40"]

   182 password: "123456"

3. key: "filebeattoredis"

4. db: 0

5. datatype: list 186

   187 ########重点重点###########

   188

   189

   190

   191 #================================ Processors

192

193 # Configure processors to enhance or manipulate events generated by the
beat. 194

6. processors:

7. - add_host_metadata: ~

8. - add_cloud_metadata: ~ 198

   199 #================================ Logging

200

9. # Sets log level. The default log level is info.

10. # Available log levels are: error, warning, info, debug

11. #logging.level: debug 204

12. # At debug level, you can selectively enable logging only for some
    components.

13. # To enable all selectors use ["*"]. Examples of other selectors are
    "beat",

14. # "publish", "service".

15. #logging.selectors: ["*"] 209

    210 #============================== Xpack Monitoring

16. # filebeat can export internal metrics to a central Elasticsearch
    monitoring

17. # cluster. This requires xpack monitoring to be enabled in Elasticsearch.
    The

18. # reporting is disabled by default. 214

19. # Set to true to enable the monitoring reporter.

20. #xpack.monitoring.enabled: false 217

21. # Uncomment to send the metrics to Elasticsearch. Most settings from the

22. # Elasticsearch output are accepted here as well. Any setting that is not
    set is

23. # automatically inherited from the Elasticsearch output configuration, so
    if you

24. # have the Elasticsearch output configured, you can simply uncomment the

25. # following line.

26. #xpack.monitoring.elasticsearch:

27. 225 #================================= Migration

226

28. # This allows to enable 6.7 migration aliases

29. #migration.6_to_7.enabled: true

requirepass 123456 #480 line

#启动服务

#systemctl enable redis #systemctl start redis

#验证

#redis-cli -h 192.168.122.40 -a 123456

>keys *

#61 line

bind 0.0.0.0

#cat /etc/redis.conf

在redis服务器配置

}

elasticsearch {

>   hosts => ["http://192.168.122.20:9200","http://192.168.122.30:9200"]

>   index => "filebeattoredis-logstashfromredis-%{+YYYY.MM.dd}"

}

stdout { codec=> rubydebug }

filter {

}

output {

#在logstash服务器配置

[root@logstash ~]# cat /etc/logstash/conf.d/logstash_from_redis.conf input {

>   redis {

>   host => "192.168.122.40"

>   port => 6379

>   password => "123456"

>   db => "0"

>   data_type => "list"

>   key => "filebeattoredis"

>   }

}

生产案例

通过elk系统分析nginx日志

1 # yum -y install nginx

log_format json ‘{ "@timestamp": "$time_iso8601", ‘ ‘"remote_addr":
"$remote_addr", ‘ ‘"remote_user": "$remote_user", ‘ ‘"body_bytes_sent":
"$body_bytes_sent", ‘ ‘"request_time": "$request_time", ‘

‘"status": "$status", ‘ ‘"request_uri": "$request_uri", ‘

‘"request_method": "$request_method", ‘ ‘"http_referer": "$http_referer",
‘ ‘"http_x_forwarded_for": "$http_x_forwarded_for", ‘ ‘"http_user_agent":
"$http_user_agent"}‘;

access_log /var/log/nginx/access.log json;

#访问并查看日志

main ‘$remote_addr - $remote_user [$time_local] "$request" ‘ ‘$status
$body_bytes_sent "$http_referer" ‘

‘"$http_user_agent" "$http_x_forwarded_for"‘;

http {

log_format

#对nginx日志格式化

# systemctl start redis ; systemctl enable redis ; systemctl status redis

# redis-cli -h 192.168.122.40 -a 123456

192.168.122.40:6379> keys *

1) "filebeattoredis"

192.168.122.40:6379> llen filebeattoredis

(integer) 5

#480 line

requirepass 123456

# cat /etc/redis.conf | grep -v "#" | grep -v "^$"

bind 192.168.122.40 #61 line

# cat /etc/filebeat/filebeat.yml | grep -v "#" | grep -v "^$"
filebeat.inputs:

- type: log enabled: true paths:

- /var/log/nginx/access.log filebeat.config.modules:

path: ${path.config}/modules.d/*.yml reload.enabled: false

output.redis:

hosts: ["192.168.122.40"]

password: "123456" key: "filebeattoredis" db: 0

datatype: list processors:

• add_host_metadata: ~

• add_cloud_metadata: ~

# systemctl start filebeat ; systemctl enable filebeat ; systemctl status
filebeat

#使用filebeat收集日志，存在redis中

# yum -y install filebeat

#配置logstash，传入es

1. ]# cat /etc/logstash/conf.d/logstash_from_redis.conf

2. input {

3. redis {

   5 host => "192.168.122.40"

   6 port => 6379

   7 password => "123456"

   8 db => "0"

data_type => "list"

key => "filebeattoredis"

}

}

filter {

}

output {

elasticsearch {

hosts => ["http://192.168.122.20:9200","http://192.168.122.30:9200"]

index => "filebeattoredis-logstashfromredis-%{+YYYY.MM.dd}"

}

stdout { codec=> rubydebug }

}

]# /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f

/etc/logstash/conf.d/logstash_from_redis.conf

进入kibana进行操作

使用logstash对nginx日志进行过滤

# echo > /var/log/nginx/access.log

#清空日志

/var/log/nginx/access.log main;

access_log

main ‘$remote_addr - $remote_user [$time_local] "$request" ‘ ‘$status
$body_bytes_sent "$http_referer" ‘

‘"$http_user_agent" "$http_x_forwarded_for"‘;

http {

log_format

#nginx日志

#filebeat配置文件

1. [root@app ~]# cat /etc/filebeat/filebeat.yml

2. filebeat.inputs:

3. - type: log

4. enabled: true

5. paths:

6. - /var/log/nginx/access.log

fields: app: www

type: nginx

fields_under_root: true

output.redis:

hosts: ["192.168.122.40"]

password: "123456" key: "filebeat" db: 0

datatype: list

1 #redis

2 192.168.122.40:6379> keys *

3 1) "filebeat"

4 192.168.122.40:6379> llen filebeat

5 (integer) 1 6

#使用logstash grok插件完成对nginx日志格式化

# cat /etc/logstash/conf.d/logstash_nginx_format.conf input {

redis {

host => "192.168.122.40"

port => 6379

password => "123456"

db => "0"

data_type => "list" key => "filebeat"

}

}

filter {

if [app] == "www" {

if [type] == "nginx" { grok {

match => {

19 "message" => "%{IPV4:remote_addr} - (%{USERNAME:remote_user}|-) [%

{HTTPDATE:time_local}] "%{WORD:request_method}
%{URIPATHPARAM:request_uri} HTTP/%

{NUMBER:http_protocol}" %{NUMBER:http_status} %{NUMBER:body_bytes_sent}
"%

{GREEDYDATA:http_referer}" "%{GREEDYDATA:http_user_agent}" "(%

{IPV4:http_x_forwarded_for}|-)"" 20 }

21 overwrite => ["message"] 22 }

1. geoip {

2. source => "remote_addr"

3. target => "geoip"

4. database => "/opt/GeoLite2-City.mmdb"

5. add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]

6. add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"] 29 }

7. date {

8. locale => "en"

9. match => ["time_local", "dd/MMM/yyyy:HH:mm:ss Z"] 33 }

10. mutate {

11. convert => ["[geoip][coordinates]", "float"] 36 }

12. output {

13. elasticsearch {

    43 hosts => ["http://192.168.122.20:9200","http://192.168.122.30:9200"]

    44 index => "logstash-nginx-log-format-%{type}-%{+YYYY.MM.dd}" 45 }

    46 stdout{codec => rubydebug } 47 }

    48

    49

    50

    51

    52 # /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f

    /etc/logstash/conf.d/logstash_nginx_format.conf

kibana中操作nginx日志视图