ELK安装配置

Posted 2020-09-10 vijayfly

log服务器安装ELK,客户端安装filebeat

1.下载安装包（https://www.elastic.co/downloads）

elasticsearch-5.3.0.rpm
jdk-8u121-linux-x64.tar.gz
kibana-5.3.0-x86_64.rpm
logstash-5.3.0.rpm

2.安装JDK(要求:1.8以上)

安装步骤略..
创建软连接：ln -s /usr/local/jdk1.8/bin/java /usr/bin/

3.安装el

yum install elasticsearch-5.3.0.rpm -y
systemctl enable elasticsearch
systemctl start elasticsearch
systemctl status elasticsearch

4.安装kibana

yum install kibana-5.3.0-x86_64.rpm -y
systemctl enable kibana
vim /etc/kibana/kibana.yml
#server.port: 5601
server.port: 9001
#kibanna访问端口，根据需求改

5.安装logstash

yum install logstash-5.3.0.rpm -y
systemctl start logstash
systemctl status logstash
systemctl enable logstash

cat /etc/logstash/conf.d/log_collect.conf

input {
    beats {
      port => 5000
      type => "logs"
      codec => multiline {
        pattern => "^%{MONTHNUM}|%{IPV4}"
        negate => true
        what => previous
      }

    }
}
#

"^%{MONTHNUM}|%{IPV4}" --> 合并以月份或者IP开头的行

output {
  if [type] == "tomcat_log_api01"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "api01-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_api02"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "api02-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider01"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider01-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider02"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider02-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider03"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider03-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider04"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider04-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider05"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider05-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_provider06"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "provider06-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_fk01"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "fk01-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_fk02"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "fk02-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_admin01"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "admin01-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "tomcat_log_admin02"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "admin02-tomcatlog-%{+YYYY.MM.dd}"
    }
  }

  if [type] == "nginx_access_log01"{
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "ngx-accesslog01-%{+YYYY.MM.dd}"
    }
  }




  stdout { codec => rubydebug }
}

#index =>"api01xxx-%{+YYYY.MM.dd}"  --> 建立索引，很重要，用于kibanna索引的建立

　　

6.日志端安装filebeat

yum install filebeat-5.3.0-x86_64.rpm 
systemctl start filebeat
systemctl status filebeat
systemctl enable filebeat

vim /etc/filebeat/filebeat.yml

#output.logstash:                                     -->注释
  # The Logstash hosts
  #hosts: ["localhost:5044"]                          -->注释 

  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

#底部添加下列内容

filebeat:
  spool_size: 1024
  idle_timeout: 5s
  registry_file: .filebeat
  config_dir: /etc/filebeat/conf.d
output:
  logstash:
    hosts:
    - 10.30.205.80:5000      -->elk服务器地址
    enabled: true
shipper: {}
logging: {}
runoptions: {}

　　

vim /etc/filebeat/conf.d/tomcat_log.yml

filebeat:
  prospectors:
    - paths:
      - /usr/local/tomcat8/logs/catalina-daemon.out     --> 要收集的日志
      encoding: plain
      fields_under_root: false
      input_type: log
      ignore_older: 24h
      document_type: tomcat_log_api01                   --> 指定日志的类型，用于日志分类
      scan_frequency: 10s
      harvester_buffer_size: 16384
      tail_files: false
      force_close_files: false
      backoff: 1s
      max_backoff: 1s
      backoff_factor: 2
      partial_line_waiting: 5s
      max_bytes: 10485760

7.打开kibanna