elk+redis+filebeat

Posted 2021-11-25 hxqxiaoqi

#安装elasticsearch-5.3.1.tar.gz(单节点安装)
tar xf elasticsearch-5.3.1.tar.gz
cd elasticsearch-5.3.1

vim config/elasticsearch.yml #修改配置文件
path.data: /data/elas/data #数据目录
path.logs: /data/elas/logs
network.host: 0.0.0.0 #允许哪个IP访问
http.port: 9200

修改系统参数以确保系统有足够资源启动ES
设置内核参数
vim /etc/sysctl.conf
# 增加以下参数
vm.max_map_count=655360
执行以下命令，确保生效配置生效：
sysctl -p

设置资源参数
vim /etc/security/limits.conf
# 修改
* soft nofile 65536
* hard nofile 131072
* soft nproc 65536
* hard nproc 131072

设置用户资源参数
vim /etc/security/limits.d/20-nproc.conf
elk soft nproc 65536 # 设置elk用户参数

添加启动用户，设置权限
启动ElasticSearch5版本要非root用户，需要新建一个用户来启动ElasticSearch
useradd elk
mkdir -pv /data/elas/{data,logs} # 创建数据和日志目录
chown -R elk:elk /data/elas/ # 修改文件所有者
chown -R elk:elk /usr/local/elasticsearch-5.3.1/
su elk
bin/elasticsearch -d #-d 为保持后台运行
curl localhost:9200 来检查ES是否启动成功

#安装kibana-5.3.1-linux-x86_64.tar.gz
tar xf kibana-5.3.1-linux-x86_64.tar.gz

配置kibana
vi config/kibana.yml
server.port: 5601 #开启默认端口5601
server.host: “192.168.182.100” #站点地址,必须是IP
elasticsearch.url: http://192.168.182.100:9200 #指向>elasticsearch服务的ip地址,必须是IP
kibana.index: “.kibana”
bin/kibana #启动kibana
curl http://192.168.182.100:5601

#redis5.0.2安装
下载：wget http://download.redis.io/redis-stable.tar.gz
yum -y install gcc
tar xf redis-5.0.2.tar.gz
mv redis-5.0.2 /usr/local/redis
cd /usr/local/redis
make
make install

vim redis.conf
bind 127.0.0.1 #表示只可以本机访问，要是远程访问需要注释掉（前面加#号）
protected-mode yes #改为no 可以不用输入密码登陆
port 6379 #修改端口
daemonize no #改为 yes 后台运行

#启动和关闭redis
./src/redis-server ./redis.conf
./src/redis-cli -p 6379 shutdown

#登录
./src/redis-cli -h 127.0.0.1 -p 6379

#开机启动
chmod u+x /etc/rc.d/rc.local
echo "/usr/local/redis/src/redis-server /usr/local/redis/redis.conf" >> /etc/rc.d/rc.local

#问题
#会发现warning警告，WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
net.core.somaxconn= 1024 #sysctl.conf
vm.overcommit_memory=1 #sysctl.conf
protected-mode no #关闭保护模式

#安装filebeat-6.6.1-linux-x86_64.tar.gz
tar xf filebeat-6.6.1-linux-x86_64.tar.gz
vim filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/*.log
output.redis:
hosts: ["192.168.182.100:6379"]
key: "nginx-log"
db: 0
timeout: 5
./filebeat -e -c ./filebeat.yml #启动

#安装

input {
redis {
data_type => "list"
key => "nginx-log"
host => "192.168.182.100"
port => 6379
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:x_forwarded_for}"}
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
}
}
output {
elasticsearch {
hosts => "192.168.182.100:9200"
}
stdout { codec => rubydebug }
}

/usr/local/logstash-5.3.1/bin/logstash -f /usr/local/logstash-5.3.1/config/logstash.conf