Linux??????ELK?????????????????????FIlebeat+Redis+Logstash+Elasticse

Posted

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Linux??????ELK?????????????????????FIlebeat+Redis+Logstash+Elasticse相关的知识，希望对你有一定的参考价值。

???????????????   conf   ??????   over   ???????????????   ??????   ??????   arch   centos6   

Centos7??????ELK??????????????????

??????ELK?????????

ELK??????????????????????????????????????????Elasticsearch???Logstash ??? Kibana???ELK???????????????????????????????????????????????????????????????????????????????????????

  • Elasticsearch: ?????????????????????????????????????????????????????????????????????????????? ????????????????????????Elasticsearch??????????????????????????????

  • Logstash??? ???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????

  • Kibana??? ????????????????????????????????????Elasticsearch???????????????????????????????????????????????????????????????????????????

  • Filebeat: ?????????????????????????????????????????????????????????????????????????????????????????????Filebeat,??????????????????????????????,Filebeat???????????????????????????????????????logstash?????????????????????????????????Elasticsearch?????????

  • Redis???NoSQL?????????(key-value)???????????????????????????????????????????????????????????????????????????????????????????????????????????????

??????ELK???????????????

??????????????????

??????????????????logstash????????????????????????????????????????????????????????????????????????????????????????????????????????????Elasticsearch?????????kibana?????????????????????????????????

??????ELK??????

??????????????????

Filebeats????????????????????????????????????????????????????????????????????????????????????????????????????????????elk?????????Filebeats??????????????????????????????Logstash??????????????????Filebeats???Logstash?????????????????????????????????????????????????????????ssl????????????????????????????????????????????????Elasticsearch???????????????kibana??????????????????

????????????ELK??????????????????

???????????????

?????? IP ????????????
1 192.168.3.206 filebeat
2 192.168.3.205 redis???logstash???elasticsearch???kibana

????????????????????????????????????????????????
https://pan.baidu.com/s/1w02WtUAqh9yX4TChyMLa5Q ?????????g0p9

  1. ???????????????filebeat???

    yum -y install filebeat
    #??????????????????????????????
rpm -qc filebeat
  2. ?????????????????????filebeat?????????????????????redis???
    ????????????????????????spring cloud?????????eureka???????????????????????????????????????????????????
vim /etc/filebeat/filebeat.yml
#????????????????????????????????????
enabled???true
paths?????????????????????
output.redis?????????????????????
                    hosts???redis???????????????IP
                    port???redis??????
                    key???redis??????key

??????????????????

  1. ????????????redis???

??????redis????????????

tar zxf redis-3.2.9.tar.gz ???C /usr/local/src

??????redis???

cd /usr/local/src/redis-3.2.9
make && make install
ln ???s /usr/local/src/redis-3.2.9 /usr/local/redis

??????redis????????????????????????????????????????????????????????????????????????????????????????????????Error????????????????????????????????????????????????????????????

??????redis???????????????

vim /usr/local/redis/redis.conf
#?????????????????????
daemonize yes                           #??????????????????
timeout 120                                #????????????
bind 0.0.0.0                                #????????????IP???????????????redis
protected-mode no                     #??????redis????????????????????????????????????????????????redis??????????????????

????????????????????????????????????????????????elk???????????????????????????????????????????????????

  1. ????????????redis?????????????????????????????????
    ??????????????????

  2. ??????filebeat??????redis???????????????????????????

??????filebeat???

systemctl start filebeat

  1. ??????redis????????????????????????

    #???????????????
keys *                          #????????????key??????????????????????????????redis????????????????????????????????????????????????
lrange eureka-log 0 -1 #??????key??????????????????filebeat???????????????????????????????????????

    ??????????????????

  2. ??????jdk1.8???

??????jdk??????????????????????????????

tar zxf /usr/local/src/jdk-8u131-linux-x64.tar.gz ???C /usr/local/
ln -s /usr/local/jdk1.8.0_91/ /usr/local/jdk

?????????????????????

vim /etc/profile
#?????????????????????
JAVA_HOME=/usr/local/jdk
export JRE_HOME=/usr/local/jdk/jre
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

???????????????????????????

source /etc/profile

??????jdk?????????????????????

java -version

??????????????????

  1. ??????Elasticsearch???

???????????????????????????

unzip elasticsearch-5.6.3.zip -d /usr/local/
mv /usr/local/ elasticsearch-5.6.3 /usr/local/elasticsearh

??????ES???????????????

vim /usr/local/elasticsearch/config/elasticsearch.yml

#??????????????????????????????????????????????????????????????????????????????????????????ES??????????????????????????????????????????
cluster.name: my-application
node.name: node-1

#????????????????????????
path.data: /opt/elk/data
path.logs: /opt/elk/logs

#ES??????????????????IP????????????
network.host: 0.0.0.0
http.port: 9200

#?????????????????????????????????????????????????????????
discovery.zen.ping.unicast.hosts: [???192.168.3.205???]

# enable cors?????????_site????????????????????????es    
http.cors.enabled: true             #????????????
http.cors.allow-origin: ???*???         #????????????

# Centos6?????????SecComp??????ES5.2.0??????bootstrap.system_call_filter???true???????????????????????????????????????????????????????????????ES????????????
bootstrap.memory_lock: false        #????????????
bootstrap.system_call_filter: false     #????????????

??????ES???????????????????????????????????????????????????????????????????????????,???????????????????????????????????????

  1. ?????????????????????
vim /etc/sysctl.conf
#????????????
vm.max_map_count=655360

?????????????????????

sysctl ???p
  1. ?????????????????????
vim /etc/security/limits.conf
#??????

*   soft    nofile  65536
*   hard        nofile  131072  
*   soft        nproc   65536
*   hard        nproc   131072

??????
??????????????????

  1. ???????????????????????????
vim /etc/security/limits.d/20-nproc.conf
#??????
elk     soft    nproc       65536

  1. ????????????????????????

    useradd elk
groupadd elk
useradd elk -g elk
  2. ???????????????????????????????????????????????????
mkdir ???pv /opt/elk/{data,logs}
chown ???R elk:elk /opt/elk
chown ???R elk:elk /usr/local/elasticsearch
  1. ???????????????????????????ES???(elk??????????????????????????????????????????elk?????????????????????)
su elk
nohup /opt/app/elasticsearch-5.6.3/bin/elasticsearch >> /dev/null 2>&1 &
  1. ??????ES?????????
    ????????????
    curl ???http://[ES IP]:9200/_search?pretty???

????????????
#???????????????
http://[ES IP]:9200/_search?pretty

  1. ??????logstash???

???????????????????????????

tar /usr/local/src/logstash-5.3.1.tar.gz ???C /usr/local/
ln ???s /usr/local/logstash-5.3.1 /usr/local/logstash

??????logstash???????????????

/usr/local/logstash/bin/logstash -e ???input { stdin { } } output { stdout {} }???

??????????????????

???????????????????????????????????????

vim /usr/local/logstash/config/logstash-simple.conf
#???????????????
input { stdin { } }
output {
    stdout { codec=> rubydebug }
}

??????logstash??????-f?????????????????????????????????

/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/logstash-simple.conf

??????????????????

?????????????????????logstash??????????????????????????????????????????????????????

  1. ????????????????????????redis??????????????????

?????????????????????

vim /usr/local/logstash/config/redis-spring.conf 
input {
  redis {
    port => "6379"
    host => "192.168.3.205"
    data_type => "list"
    type => "log"
    key => "eureka-log"
  }
}
output {
  elasticsearch {
     hosts => "192.168.3.205:9200"
     index => "logstash1-%{+YYYY.MM.dd}"
  }
}

?????????????????????????????????????????????

/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/redis-spring.conf

???????????????
??????????????????

????????????????????????reids???key???????????????????????????????????????????????????logstash?????????
??????????????????

  1. ??????curl ??????ES?????????????????????
curl http://192.168.3.205:9200/_search?pretty

???????????????
??????????????????

??????????????????logstash???redis?????????????????????????????????ES??????ok??????

  1. ??????ES????????????elasticsearch-head???

??????head???????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????

????????????
??????node-v8.2.1.tar.gz phantomjs-2.1.1-linux-x86_64.tar.bz2 ?????????
??????node???
tar zxvf node-v8.2.1.tar.gz
cd node-v8.2.1/
./configure && make && make install 

??????phantomjs???
tar jxvf phantomjs-2.1.1-linux-x86_64.tar.bz2
cd phantomjs-2.1.1-linux-x86_64/bin/
cp phantomjs /usr/local/bin/

??????es-head?????????????????????
unzip master.zip ???d /usr/local/
cd elasticsearch-head/
npm install
npm run start &

????????????????????????????????????9100???
netstat ???anpt | grep 9100

????????????
git clone git://github.com/mobz/elasticsearch-head.git
cd elasticsearch-head
npm install
npm run start
netstat ???anpt | grep 9100

????????????
????????????
docker push mobz/elasticsearch-head:5
???????????????
docker run -p 9100:9100 mobz/elasticsearch-head:5
web???????????????
http://IP:9100
  1. Elasticsearch-head????????????Web?????????????????????
    ??????????????????

???????????????logstash??????ES???????????????
??????????????????

  1. ??????kibana

???????????????kibana???

tar -zxvf /usr/local/src/kibana-5.3.1-linux-x86_64.tar.gz -C /usr/local/

??????kibana???????????????

vim /usr/local/kibana-5.3.1-linux-x86_64/config/kibana.yml
?????????????????????
server.port: 5601                                                            #??????????????????5601
server.host: ???192.168.3.205???                                    #kibana??????IP
elasticsearch.url: http://192.168.3.205:9200        #??????ES????????????IP Port
kibana.index: ???.kibana???

????????????kibana???

nohup /usr/local/kibana-5.3.1-linux-x86_64/bin/kibana >> /dev/null 2>&1 &

?????????????????????

netstat ???anot | grep 5601

??????????????????????????????kibana???????????????
??????????????????

??????Web??????kibana???

http://[Kibana IP]:5601

?????????????????????????????????????????????????????????????????????????????????????????????
??????????????????

??????logstash???????????????index???????????????
????????????logstash??????index???
??????????????????

Kibana?????????index???
??????????????????

????????????1,2,3,4?????????????????????
??????????????????

?????????????????????Discover??????????????????????????????????????????
??????????????????

???????????????ELK?????????OK????????????????????????????????????

以上是关于Linux??????ELK?????????????????????FIlebeat+Redis+Logstash+Elasticse的主要内容，如果未能解决你的问题，请参考以下文章

Linux企业运维——ELK日志分析平台（上）安装与配置ES图形化管理插件

Linux------------ELK日志收集系统入门

Linux里面ELK作用是啥？

Linux下开源实时日志分析elk部署详解

Linux一键部署ELK日志平台自动化脚本

linux12Elk --＞02elasticsearch部署