安装 fluent-bit ，以及导入日志目录到es中

Posted 2020-10-07

#!/bin/bash
# by gaogd 
esname=internal-2013865141.cn-north-1.elb.amazonaws.com.cn
esport=9200
fluentdir=/work/opt/fluentb/build/conf
logdir=/data/logs/
host=`hostname`

function  createfluentbconf ()
{
if [ ! -d  $logdir/fluent  ]
then
        mkdir -p $logdir/fluent
fi
if [ ! -d  $fluentdir/build/conf  ]
then
        mkdir -p $fluentdir/build/conf
fi
cat > /etc/supervisor/conf.d/fluentb.conf <<EOF
[program:fluentb]
directory=/work/opt/fluentb/build
command=/work/opt/fluentb/build/bin/fluent-bit -c /work/opt/fluentb/build/conf/fluent.conf
;process_name=%(process_num)02d
numprocs=1
user=root
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/data/logs/fluent/fluent.log
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
EOF


cat >${fluentdir}/fluent.conf<<EOF
[SERVICE]
    Flush        1
    Daemon       Off
    Log_Level    info

###INPUT
###OUTPUT

EOF


for name in `ls ${logdir}`
do
cat >${fluentdir}/input-${name}.conf<<EOF
[INPUT]
    Name        tail
    Path        /data/logs/${name}/*.log
    Db       /data/logs/${name}/${name}.pos
    Tag         ${name}
    Mem_Buf_Limit 5MB

EOF

cat >${fluentdir}/output-${name}.conf<<EOF
[OUTPUT]
    Name  es
    Match ${name}
    Host  ${esname}
    Port  ${esport}
    Index fluentd-${name}
    Logstash_Format On
    Logstash_Prefix fluentd-${name}
    Include_Tag_Key On
EOF
#sed -i "/###INPUT/a\@INCLUDE ${fluentdir}/input-${name}.conf" ${fluentdir}/fluent.conf
#sed -i "/###OUTPUT/a\@INCLUDE ${fluentdir}/output-${name}.conf" ${fluentdir}/fluent.conf
done
cat ${fluentdir}/input-* >>${fluentdir}/fluent.conf
cat ${fluentdir}/output-*>>${fluentdir}/fluent.conf

cat >>${fluentdir}/fluent.conf<<EOF
[FILTER]
    Name record_modifier
    Match *
    Record hostname "${host}"
EOF
}



function installfluentd ()
{   
    apt-get install gcc cmake g++ openssl -y
    rm -rf /work/opt/fluent*
    mkdir -p /work/{source,opt}
    cd /work/source/
    mv /tmp/fluent-bit-0.12.0.tar.gz  .
    tar xf fluent-bit-0.12.0.tar.gz
    mv fluent-bit-0.12.0 /work/opt/
    cd /work/opt/
    /bin/ln -s /work/opt/fluent-bit-0.12.0/ fluentb
    mkdir -p  /work/opt/fluentb/build  && cd /work/opt/fluentb/build && cmake ../ && make && make install
}

installfluentd
createfluentbconf

本文出自 “奋斗吧” 博客，请务必保留此出处http://lvnian.blog.51cto.com/7155281/1966607