ELK

Posted xcbki

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了ELK相关的知识，希望对你有一定的参考价值。

#环境 centos 7.4 ,ELK 6 ,单节点
 
#服务端
Logstash 收集,过滤   
Elasticsearch 存储,索引日志
Kibana 可视化
 
#客户端
filebeat 监控、转发,作为agent
filebeat-->Logstash-->Elasticsearch-->Kibana
 
#内核
echo ‘
* hard nofile 65536
* soft nofile 65536
* soft nproc 65536
* hard nproc 65536
‘>>/etc/security/limit.conf
echo ‘ vm.max_map_count = 262144
net.core.somaxconn=65535
net.ipv4.ip_forward = 1
‘>>/etc/sysctl.conf
sysctl -p
 
#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i ‘/^SELINUX=.*/c SELINUX=disabled‘ /etc/selinux/config
sed -i ‘s/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g‘ /etc/selinux/config
grep --color=auto ‘^SELINUX‘ /etc/selinux/config
setenforce 0
 
 
#配置yum源
echo ‘
[elk-6]
name=elk-6
gpgcheck=1
enabled=1
‘>/etc/yum.repos.d/elk.repo
 
 
安装
yum install java-1.8.0-openjdk -y
yum install elasticsearch -y
yum install logstash -y
yum install kibana -y
 
elasticsearch配置
cp /etc/elasticsearch/elasticsearch.yml{,.bak}
echo ‘
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
cluster.name: ELK
node.name: elk.novalocal 
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["192.168.1.30:9300"]
discovery.zen.minimum_master_nodes: 1 
‘>/etc/elasticsearch/elasticsearch.yml
 
systemctl daemon-reload
systemctl enable elasticsearch
systemctl restart elasticsearch
systemctl status elasticsearch
 
logstash配置
cp /etc/logstash/logstash.yml{,.bak}
echo ‘path.config: /etc/logstash/conf.d‘>>/etc/logstash/logstash.yml
#添加一个日志处理文件
#filebeat->logstash->elasticsearch
echo ‘
input {
 
#收集本地log#
  file {
     type => "logtest"
     path => "/var/log/logtest.txt"
     start_position => "beginning"
  }
 
#filebeat客户端#
  beats {
     port => 5045
  }
}
#筛选
#filter { }
 
output {
 
#标准输出，调试使用#
  stdout {
   codec => rubydebug { }
  }
 
# 输出到es#
  elasticsearch {
    hosts => ["http://192.168.1.30:9200"]
    index => "%{type}-%{+YYYY.MM.dd}"
  }
 
}
‘>/etc/logstash/conf.d/logstash-01.conf
 
#检测配置 
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf --config.test_and_exit
 
#生成测试
echo $(date +"%F-%T") log-test >>/var/log/logtest.txt
 
#启动，查看生成日志
nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf &
 
 
kibana配置
cp /etc/kibana/kibana.yml{,.bak}
echo ‘
server.port: 5601
server.host: "0.0.0.0"
# ES的url的一个ES节点#
elasticsearch.url: "http://192.168.1.30:9200"
kibana.index: ".kibana"
#kibana.defaultAppId: "home"
‘>/etc/kibana/kibana.yml
 
#启动 
systemctl enable kibana
systemctl restart kibana
 
客户端安装
yum install -y filebeat
 
配置filebeat收集nginx日志
echo ‘
#filebeat#
filebeat.prospectors:
#nginx
- input_type: log
  enable: yes
  #tags: nginx-access
  paths:
    - /usr/local/nginx/logs/access.log
  exclude_lines: ["^$"]
  fields:
    type: "nginx-access"
  fields_under_root: true
 
 
output.logstash:
  hosts: ["10.0.0.30:5044"]
  #hosts: ["172.16.50.32:5044"]
  #index: filebeat
‘>/etc/filebeat/filebeat.yml
 
启动
systemctl enable filebeat
systemctl restart filebeat
systemctl status filebeat
 
 
 
 
 
 
nginx默认日志格式
log_format main    ‘$remote_addr - $remote_user [$time_local] "$request" ‘ ‘$status $body_bytes_sent                                   "$http_referer" ‘ ‘"$http_user_agent" "$http_x_forwarded_for"‘;
 
创建nginx正则表达式(引用grok正则)
echo ‘#nginx-access
WZ ([^ ]*)
NGINXACCESS %{IP:remote_ip} - - [%{HTTPDATE:timestamp}] "%{WORD:method} %{WZ:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:bytes} %{QS:referer} %{QS:agent} %{QS:xforward}
‘>/etc/logstash/conf.d/nginx-access
 
重新生成logstash配置文件
echo ‘
input {
 
#收集本地log#
  file {
     type => "logtest"
     path => "/var/log/logtest.txt"
     start_position => "beginning"
  }
 
#filebeat客户端#
  beats {
     port => 5044
  }
 
}
 
# #筛选
filter {
 
# 如果是nginx访问日志
  if ( [type] == "nginx-access" ) {
 
    #按字段切割
    grok {
      patterns_dir=>"/etc/logstash/conf.d/nginx-access"
      match => { "message" => "%{NGINXACCESS}" }
      }
 
    # 时间格式转换
    date {
      match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
      }
 
    # 删除不需要的字段
    mutate {
      remove_field => [ "offset", "@version", "beat", "input_type", "tags","id"]
      }
    }
}
 
output {
 
#标准输出，调试使用#
  stdout {
   codec => rubydebug { }
  }
 
# 输出到es#
  elasticsearch {
    hosts => ["http://172.16.50.32:9200"]
    index => "%{type}-%{+YYYY.MM.dd}"
  }
 
}
‘>/etc/logstash/conf.d/logstash-01.conf
 
 
检测配置
/usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/logstash-01.conf
 
调试logstash
#关闭
systemctl stop  logstash
#在终端启动查看
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-01.conf
 
 
修改时区
timedatectl list-timezones |grep Shanghai #查找中国时区的完整名称
Asia/Shanghai
timedatectl set-timezone Asia/Shanghai #其他时区以此类推
 
ntpdate 0.asia.pool.ntp.org
 
 
 

以上是关于ELK的主要内容，如果未能解决你的问题，请参考以下文章

ELK企业级日志分析系统

ELK企业级日志分析系统

elk是啥

ELK系列：.net core中使用ELK

elk-准备

ELK