每次有新服务器上线的时候我们需要采集日志信息涉及到各种配置：
- rsyslog客户端
- rsyslog中继服务器
- rsyslog服务器
- logstash-shipper
如果手动配置的话工作量较大，而且容易出错。写了个脚本以便快速部署。
文件结构如下
? elk_scripts git:(master) ? tree . ├── README ├── log-make.sh ├── rsyslog-clinet-temp └── rsyslog-server-temp
执行log-make.sh将询问操作内容，1为创建配置文件，2为发送配置文件
[[email protected] elk_scripts]# sh log-make.sh #####################choose number################################# What do you want to do? 1.Make ELK configuration! 2.Transfer configuration! #####################choose number#################################
各文件内容如下
log-make.sh
#!/bin/bash
RED_COLOR=‘\E[1;31m‘
GREEN_COLOR=‘\E[1;32m‘ YELLOW_COLOR=‘\E[1;33m‘ RES=‘\E[0m‘ NGATAG=`hostname`-nginx-access NGETAG=`hostname`-nginx-error phpSLOWTAG=`hostname`-php-slow PHPERRORTAG=`hostname`-php-error ACTIONTAG=`hostname`-action BRANCH=`hostname|awk -F ‘-‘ ‘{print $1}‘` [ -d ./logfile ]||mkdir ./logfile rsyslog_client () { if [ ! -f /var/log/nginx/access-admin.log ];then echo -e "${RED_COLOR}Can not find access-admin log.${RES}" exit 1 fi if [ ! -f /var/log/nginx/error-admin.log ];then echo -e "${RED_COLOR}Can not find access-admin error log.${RES}" exit 2 fi if [ ! -f /var/log/nginx/access-pay.log ];then echo -e "${RED_COLOR}Can not find access-pay log.${RES}" exit 3 fi if [ ! -f /var/log/nginx/error-pay.log ];then echo -e "${RED_COLOR}Can not find error-pay log.${RES}" exit 4 fi if [ ! -f /var/log/nginx/access-frontend.log ];then echo -e "${RED_COLOR}Can not find access-frontend log.${RES}" exit 11 fi if [ ! -f /var/log/nginx/error-frontend.log ];then echo -e "${RED_COLOR}Can not find error-frontend log.${RES}" exit 12 fi cp rsyslog-clinet-temp ./logfile/rsyslog-client sed -i "s#nginxaccesstag#${NGATAG}#g" ./logfile/rsyslog-client sed -i "s#nginxerrortag#${NGETAG}#g" ./logfile/rsyslog-client sed -i "s#phpslowtag#${PHPSLOWTAG}#g" ./logfile/rsyslog-client sed -i "s#phperrortag#${PHPERRORTAG}#g" ./logfile/rsyslog-client sed -i "s#actionlog#${ACTIONTAG}#g" ./logfile/rsyslog-client } rsyslog_server () { cp rsyslog-server-temp ./logfile/rsyslog-server sed -i "s#nginx-access#${NGATAG}#g" ./logfile/rsyslog-server sed -i "s#nginx-error#${NGETAG}#g" ./logfile/rsyslog-server sed -i "s#php-slow#${PHPSLOWTAG}#g" ./logfile/rsyslog-server sed -i "s#php-error#${PHPERRORTAG}#g" ./logfile/rsyslog-server sed -i "s#action-log#${ACTIONTAG}#g" ./logfile/rsyslog-server sed -i "s#ngxapath#/data/rsyslog/nginx/${BRANCH}/$(hostname)-nginx-access.log#g" ./logfile/rsyslog-server sed -i "s#ngxepath#/data/rsyslog/nginx/${BRANCH}/$(hostname)-nginx-error.log#g" ./logfile/rsyslog-server sed -i "s#phpspath#/data/rsyslog/php/${BRANCH}/$(hostname)-php-slow.log#g" ./logfile/rsyslog-server sed -i "s#phpepath#/data/rsyslog/php/${BRANCH}/$(hostname)-php-error.log#g" ./logfile/rsyslog-server sed -i "s#actionpath#/data/rsyslog/php/${BRANCH}/$(hostname)-action.log#g" ./logfile/rsyslog-server } send_rsyslog_client () { cp /etc/rsyslog.conf{,.bak_$(date +%F)} \cp ./logfile/rsyslog-client /etc/rsyslog.conf } send_rsyslog_server () { scp -P 2222 ./logfile/rsyslog-server [email protected]rsyslog中继服务器地址:/etc/rsyslog.d/`hostname`.conf scp -P 2001 ./logfile/rsyslog-server [email protected]rsyslog服务器地址:/etc/rsyslog.d/`hostname`.conf systemctl restart rsyslog if [ `systemctl status rsyslog|grep "active (running)"|wc -l` -eq 1 ] then echo "Rsyslog client servie start successfully!" else echo "Rsyslog client service start failure!" break 7 fi ssh -p 2222 [email protected]rsyslog中继服务器地址 "systemctl restart rsyslog" ssh -p 2001 [email protected]服务器地址 "systemctl restart rsyslog" } change_shipper_sitename () { sed -i "s#SITENAME#${BRANCH}#g" $0 } send_shipper_conf_tw_slave01 () { shipper_nginx_access=`ssh -p 2001 [email protected]logstash-shipper地址 "grep ${BRANCH}-nginx-access /etc/logstash-shipper/conf.d/shipper.conf|wc -l"` shipper_nginx_error=`ssh -p 2001 [email protected]logstash-shipper地址 "grep ${BRANCH}-nginx-error /etc/logstash-shipper/conf.d/shipper.conf|wc -l"` shipper_php_slow=`ssh -p 2001 [email protected]地址 "grep ${BRANCH}-php-slow /etc/logstash-shipper/conf.d/shipper.conf|wc -l"` shipper_php_error=`ssh -p 2001 [email protected]地址 "grep ${BRANCH}-php-error /etc/logstash-shipper/conf.d/shipper.conf|wc -l"` shipper_action=`ssh -p 2001 [email protected]地址 "grep ${BRANCH}-action /etc/logstash-shipper/conf.d/shipper.conf|wc -l"` if [ ${shipper_action} -eq 0 ] then ssh -p 2001 [email protected]地址 ‘sed -i "2i\ file {\n path => \"/data/rsyslog/php/SITENAME/SITENAME*action.log\"\n type => \"SITENAME-action\"\n sincedb_path => \"/data/sincedb/SITENAME\"\n }\n" /etc/logstash-shipper/conf.d/shipper.conf‘ else echo "ELK action shipper configuration added already!" continue fi if [ ${shipper_nginx_access} -eq 0 ] then ssh -p 2001 [email protected]地址 ‘sed -i "2i\ file {\n path => \"/data/rsyslog/nginx/SITENAME/SITENAME-*-nginx-access.log\"\n type => \"SITENAME-nginx-access\"\n sincedb_path => \"/data/sincedb/SITENAME\"\n }\n" /etc/logstash-shipper/conf.d/shipper.conf‘ else echo "ELK nginx-access shipper configuration added already!" continue fi if [ ${shipper_nginx_error} -eq 0 ] then ssh -p 2001 [email protected]地址 ‘sed -i "2i\ file {\n path => \"/data/rsyslog/nginx/SITENAME/SITENAME-*-nginx-error.log\"\n type => \"SITENAME-nginx-error\"\n sincedb_path => \"/data/sincedb/SITENAME\"\n }\n" /etc/logstash-shipper/conf.d/shipper.conf‘ else echo "ELK nginx-error shipper configuration added already!" continue fi if [ ${shipper_php_slow} -eq 0 ] then ssh -p 2001 [email protected]地址 ‘sed -i "2i\ file {\n path => \"/data/rsyslog/php/SITENAME/SITENAME-*-php-slow.log\"\n type => \"SITENAME-php-slow\"\n sincedb_path => \"/data/sincedb/SITENAME\"\n }\n" /etc/logstash-shipper/conf.d/shipper.conf‘ else echo "ELK php-slow shipper configuration added already!" continue fi if [ ${shipper_php_error} -eq 0 ] then ssh -p 2001 [email protected]地址 ‘sed -i "2i\ file {\n path => \"/data/rsyslog/php/SITENAME/SITENAME-*-php-error.log\"\n type => \"SITENAME-php-error\"\n sincedb_path => \"/data/sincedb/SITENAME\"\n }\n" /etc/logstash-shipper/conf.d/shipper.conf‘ else echo "ELK php-error shipper configuration added already!" fi } echo ‘#####################choose number#################################‘ echo -e "${YELLOW_COLOR}What do you want to do?${RES}" echo -e "${GREEN_COLOR}1.Make ELK configuration!${RES}" echo -e "${GREEN_COLOR}2.Transfer configuration!${RES}" echo ‘#####################choose number#################################‘ read -p "Choose number:" NUMBER case ${NUMBER} in 1) rsyslog_client rsyslog_server change_shipper_sitename echo "Log file make successfully!" ;; 2) send_rsyslog_client send_rsyslog_server send_shipper_conf_tw_slave01 ;; *) echo "Usage:{1|2}" exit 9 esac
rsyslog-clinet-temp
$ModLoad imuxsock $ModLoad imjournal $ModLoad imfile $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat $IncludeConfig /etc/rsyslog.d/*.conf *.info;mail.none;authpriv.none;cron.none /var/log/messages authpriv.* /var/log/secure mail.* -/var/log/maillog cron.* /var/log/cron *.emerg :omusrmsg:* uucp,news.crit /var/log/spooler local7.* /var/log/boot.log ##########Start Nginx Log File################# $InputFileName /var/log/nginx/access-admin.log $InputFileTag nginxaccesstag: $InputFileStateFile nginxaccesstag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileName /var/log/nginx/error-admin.log $InputFileTag nginxerrortag: $InputFileStateFile nginxerrortag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileName /var/log/nginx/access-frontend.log $InputFileTag nginxaccesstag: $InputFileStateFile nginxaccesstag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileName /var/log/nginx/error-frontend.log $InputFileTag nginxerrortag: $InputFileStateFile nginxerrortag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileName /var/log/nginx/access-pay.log $InputFileTag nginxaccesstag: $InputFileStateFile nginxaccesstag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileName /var/log/nginx/error-pay.log $InputFileTag nginxerrortag: $InputFileStateFile nginxerrortag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 ######################End Of Nginx Log File################ ######################Start Of Action Log File############# $InputFileName /var/log/php-fpm/action_log.log $InputFileTag actionlog: $InputFileStateFile actionlog $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 ######################End Of Action Log File############### #####################Start PHP Log File################### $InputFileName /var/log/php-fpm/www-slow.log $InputFileTag phpslowtag: $InputFileStateFile phpslowtag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $InputFileReadMode 2 $InputFileName /var/log/php-fpm/error.log $InputFileTag phperrortag: $InputFileStateFile phperrortag $InputFileSeverity debug $InputRunFileMonitor $InputFilePollInterval 1 $WorkDirectory /var/lib/rsyslog $ActionQueueType LinkedList $ActionQueueFileName srvrfwd $ActionResumeRetryCount -1 $ActionQueueSaveOnShutdown on ####################End Of PHP log File##################### ###################Start Log Forward################################## if $programname == ‘nginxaccesstag‘ then @@rsyslog中继服务器地址:514 if $programname == ‘nginxerrortag‘ then @@rsyslog中继服务器地址:514 if $programname == ‘phpslowtag‘ then @@rsyslog中继服务器地址:514 if $programname == ‘phperrortag‘ then @@rsyslog中继服务器地址:514 if $programname == ‘actionlog‘ then @@rsyslog中继服务器地址:514 ###################End Of log Forward################################
rsyslog-server-temp
$template nginx-access,"ngxapath" $template nginx-error,"ngxepath" $template php-slow,"phpspath" $template php-error,"phpepath" $template action-log,"actionpath" if $programname == ‘nginx-access‘ then ?nginx-access if $programname == ‘nginx-error‘ then ?nginx-error if $programname == ‘php-slow‘ then ?php-slow if $programname == ‘php-error‘ then ?php-error if $programname == ‘action-log‘ then ?action-log