线上日志分析与其他一些脚本

Posted 2020-06-10

对一些线上常用的脚本进行了一下总结和说明，免得以后忘记了~

一·线上发布API集群的代码脚本：

#!/bin/bash
#Author CCC

host=‘
10.44.22.113
10.44.22.113
10.44.22.112
10.44.22.112
10.44.22.113
10.44.22.113
10.44.22.114
10.44.22.114
10.44.22.115
10.44.22.115
10.44.22.119
10.44.22.119
‘
#basePath=‘/var/www‘--------------------定义要发布的文件路径
basePath=‘/root/xxxx/2016-01-27‘----------定义源文件的路径
check_port()---------检查端口已经已经起来了。
{

ss=0;
count=0;

ip=$2;
port=$1;
for ((i=0;i<120;i++))
do
#    $((count++))
    if [[ -n $(ssh $ip "netstat -ntlp | grep $port") ]]
    then
        ss=1;
        break;
    else
        ss=0;
    fi
sleep 1
done

return $ss;

}


check_return()---------检查返回内容
{
  if [[ "$1" == "0" ]]
  then
      echo $2
      exit
  fi
  echo $2
  host_ip=""
}

sed -i "s/HEAP_MAX=2g/HEAP_MAX=6g/g" $basePath/query/bin/properties/jvm.properties------------------------修改一些JVM参数，因为目前打包是公共打包，参数可能配置不准确
sed -i "s/PERM_SIZE=128m/PERM_SIZE=256m/g" $basePath/query/bin/properties/jvm.properties
sed -i "s/HEAP_INIT=2g/HEAP_INIT=6g/g" $basePath/query/bin/properties/jvm.properties
sed -i "s/YOUNG_GEN_SIZE=256m/YOUNG_GEN_SIZE=4g/g" $basePath/query/bin/properties/jvm.properties
for i in $host--------------循环进行每台机器的发布
do
      echo "deploy $i"
      ssh -o StrictHostKeyChecking=no $i "source /etc/profile;sh /var/www/query/bin/stop.sh"
      rsync -e "ssh -o StrictHostKeyChecking=no" -av  $basePath/query $i:/var/www/ --delete >> /dev/null 2>&1
      ssh -o StrictHostKeyChecking=no $i "source /etc/profile;sh /var/www/query/bin/restart.sh"
  check_port 82 $i
  check_return $? $i
  sleep 10

done

二、线上GET方式查看API集群中API的返回结果：

#!sbin/bash
#author CCC



providers="`cat server_ip`" ----------找到同目录底下的ip列表

for host in $providers
do 

RESULT=`curl ‘http://‘$host‘:82/productapi/xxxxxxx?bbb=23451243&22=1‘` ----------循环执行命令和请求
echo $host    $RESULT ------打印日志

done

三、查看服务器列表的时间，与上一个相似：

#!sbin/bash
#author CCC

providers="`cat server_ip`"

for host in $providers
do
ssh -o StrictHostKeyChecking=no $host "date"
done

四、集群日志分析与返回，是个用的比较多的脚本 *****：

#!/bin/bash
#Author CCC

host=‘ -----------host列表
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
‘
for i in $host 
do
      echo "deploy $i"
     # ssh $i ‘grep "java.net.UnknownHostException: xxxxxxxxx.idc2" /d1/joblog/query/dailylog.2015-12-18_09|awk "{print NR==1}"‘ ------------查到每台机器上面的xx目录下这个错误的内容，并且找到第一列的内容打印
     # ssh $i "grep find_gift /d1/joblog/query/accesslog.2015-12-30|grep 20633051|awk ‘{if(\$NF>200){print \$0}}‘" ----找到每个机器上面find_gift接口中商品id是20633051的接口返回时间大于200ms的内容并打印
     # ssh $i "grep find_xxx_lighter  /d1/joblog/query/accesslog.2016-01-12_19|grep 20633051|awk ‘{print \$NF}‘|sort|uniq -c"----找到xxxx接口中内容包含20633051商品的返回时间（最后一列）排序并计数
     # ssh $i "grep product.php /d1/joblog/query/dailylog.2015-12-28*|awk ‘{print \$NF}‘|tr -d ms|sort|uniq -c" ----找到product.php的响应时间，替换到ms关键词，排序计数
      ssh $i "grep promo_product /d1/joblog/query/accesslog.2016-01-24|grep 20260420|wc -l"----同上上
     #  ssh $i "grep ‘Request:‘ /d1/joblog/query/dailylog.2016-01-05*|grep ‘Use time:‘|awk {‘print \$NF‘}|tr -d ms|awk ‘{if(\$1>500){print \$1}}‘|wc -l"----找到外部请求超过500ms的接口并计数
     # ssh $i "sed -n ‘/^10:00:/‘,‘/^10:20:/p‘ /d1/joblog/query/accesslog.2016-01-13| grep find_lighter|awk ‘{print \$NF}‘|sort|uniq -c"----找到日志在10点到10点20之间lighter接口的响应时间
done

五、全量清理Redis：

host=‘
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
‘
ports=‘
8279
8379
8479
8579
‘
for h in $host
do
for p in $ports
do
/usr/local/bin/redis-cli -h $h -p $p FLUSHALL
sleep 10
done
done

六、整理内容发邮件：

DATE=`date -d "-1 hour" +%Y-%m-%d_%H_%M`  -------时间定义
cd /root/analyze -------目录
python query.py > /root/analyze/result_$DATE.txt----运行python脚本并输出到文件中
mail -s "aaaaa Statistics @ $DATE" [email protected] < /root/analyze/result_$DATE.txt ---发送邮件

 七、项目整体发布脚本：

#!bin/bash
#author CCC  

cd /home/cxht/svn/cxht/XXXXX-parent  -----------更新代码与打包
svn update
#/home/cxht/apache-maven-3.2.1/bin/mvn -e -U clean install package -Dmaven.test.skip=true
cd /home/cxht/svn/cxht/XXXXX-parent/XXXXX-package
/home/cxht/apache-maven-3.2.1/bin/mvn -e -U clean install -Dmaven.test.skip=true
/home/cxht/apache-maven-3.2.1/bin/mvn package

kill -9 `ps -ef|grep /home/cxht/tomcat|grep -v grep|awk ‘{print $2}‘`   ---------杀掉tomcat进程
ps -ef|grep java|grep /var/www/YYYYY/|awk ‘{print $2}‘|xargs -i kill -9 {} ---------杀掉其他main方法方式运行的程序
rm -fr /var/www/YYYYY/* --------删除所有代码

cd /var/www/YYYYY/

find  /home/cxht/svn/cxht/XXXXX-parent/* -name *.gz|xargs -i tar -xvf {} ----------找到所有的gz包解压
unzip /home/cxht/svn/cxht/XXXXX-parent/XXXXX-web/target/XXXXX-web.war -d /var/www/YYYYY/XXXXX-web --------解压war包
unzip /home/cxht/svn/cxht/XXXXX-parent/XXXXX-event-maker/target/XXXXX-event-maker-bin.zip -d /var/www/YYYYY/XXXXX-event-maker

#change jvm memory size ------------设置 一些jvm的属性
sed -i "s/HEAP_INIT=2g/HEAP_INIT=512m/g" `find /var/www/YYYYY/* -name jvm.properties`
sed -i "s/HEAP_MAX=2g/HEAP_MAX=1g/g" `find /var/www/YYYYY/* -name jvm.properties`

#start ------重启

find /var/www/YYYYY/* -name restart.sh|xargs -i sh {}
sh /home/cxht/tomcat/bin/startup.sh
sh /var/www/YYYYY/XXXXX-event-maker/XXXXX-event-maker/bin/main.sh restart