ElasticSearch异常情况监控处理

Posted 2022-08-01 张志翔 ̮

1.异常类型

在公司使用Canal-Server作为数据同步监听,在同步过程中,会出现各种问题异常,

比如常见一下几种:

1.表结构发生变更,导出BinLog字段对比异常

2.数据库连接失败,账号密码错误, 权限不足等

3.Binlog文件过期或者丢失

2.传统监控手段-ELK

1.应用端按固定格式,把日志写入到log文件中

2.logstash读取日志文件, 写入到ES中

3.通过logstash分析日志,按特定规则配置告警规则,发送钉钉报警

3.轻量级监控手段logback

1.自定义LogDingDingAppende

package com.logtest.appender;


import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.PatternLayout;
import ch.qos.logback.classic.spi.LoggingEvent;
import ch.qos.logback.core.UnsynchronizedAppenderBase;
import com.alibaba.fastjson.JSON;
import com.logtest.appender.endity.DingtalkMessage;
import com.logtest.util.HttpsUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;

import java.util.HashMap;
import java.util.Map;



@Slf4j
public class LogDingDingAppender extends UnsynchronizedAppenderBase<LoggingEvent> 


    private String pattern = "%level\\n%m\\n%logger\\n%ex";
    private PatternLayout layout;
    private String profilesActive = "unknown";
    private String dingdingUrl;
    private Level nowLevel = Level.ERROR;
    private String secret;
    private String serverName;

    /**
     * 排除的类集合用逗号分隔
     */
    private String excludeThrowableClasses;

    public String getProfilesActive() 
        return profilesActive;
    

    public void setProfilesActive(String profilesActive) 
        this.profilesActive = profilesActive;
    

    public void setPattern(String pattern) 
        this.pattern = pattern;
    

    public String getServerName() 
        return serverName;
    

    public void setServerName(String serverName) 
        this.serverName = serverName;
    

    public String getPattern() 
        return pattern;
    

    public PatternLayout getLayout() 
        return layout;
    

    public void setLayout(PatternLayout layout) 
        this.layout = layout;
    

    public String getDingdingUrl() 
        return dingdingUrl;
    

    public void setDingdingUrl(String dingdingUrl) 
        this.dingdingUrl = dingdingUrl;
    

    public void setLevel(String level) 
        this.nowLevel = Level.toLevel(level, Level.ERROR);
    

    public void setExcludeThrowableClasses(String excludeThrowableClasses) 
        this.excludeThrowableClasses = excludeThrowableClasses;
    

    public String getSecret() 
        return secret;
    

    public void setSecret(String secret) 
        this.secret = secret;
    


    @Override
    protected void append(LoggingEvent eventObject) 
        try 

            if (eventObject.getLevel().isGreaterOrEqual(nowLevel)) 
                if (excludeThrowableClasses != null && eventObject.getThrowableProxy() != null) 
                    if (excludeThrowableClasses.contains(eventObject.getThrowableProxy().getClassName())) 
                        return;
                    
                
                String toMsg = layout.doLayout(eventObject);
                sendMarkDownMsg("Canal-Server监听异常", toMsg);
            
         catch (Exception e) 
            e.printStackTrace();
        
    

    @Override
    public void start() 
        //sendStartMsg();
        System.out.println("Canal-Server开始监控,应用名:" + getServerName());
        PatternLayout patternLayout = new PatternLayout();
        patternLayout.setContext(context);
        patternLayout.setPattern(getPattern());
        patternLayout.start();
        this.layout = patternLayout;
        super.start();


    


    /**
     * @Author
     * @Description 开始监听
     * @Date 2021-09-29 18:05
     * @Param
     * @return
     **/
    public void sendStartMsg() 

        String title = "Canal-Server开始监控";
        String prefixEnd = "</font> \\n";
        String prefixStart = "####";
        String msg = "### " + title + prefixEnd +
                prefixStart + " 应用名：<font color=#1A73E8 size=2>" + getServerName() + prefixEnd;

        DingtalkMessage message = new DingtalkMessage();
        DingtalkMessage.Markdown markdown = new DingtalkMessage.Markdown();
        markdown.setTitle(title);
        markdown.setText(msg);
        message.setMarkdown(markdown);
        Map<String, String> header = new HashMap<>();
        header.put("Content-Type", "application/json");
        String sendMsgRet = HttpsUtils.post(dingdingUrl, JSON.toJSONString(message), header);
        log.info("钉钉机器人返回:" + sendMsgRet);

    

    /**
     * @Author
     * @Description 钉钉发送信息
     * @Date 2021-09-29 18:04
     * @Param
     * @return
     **/
    public void sendMarkDownMsg(String title, String msg) 

        DingtalkMessage message = new DingtalkMessage();
        DingtalkMessage.Markdown markdown = new DingtalkMessage.Markdown();
        markdown.setTitle(title);
        markdown.setText(getMarkdown(title, msg));
        message.setMarkdown(markdown);
        Map<String, String> header = new HashMap<>();
        header.put("Content-Type", "application/json");
        String sendMsgRet = HttpsUtils.post(dingdingUrl, JSON.toJSONString(message), header);
        log.info("钉钉机器人返回:" + sendMsgRet);

    

    /**
     * @Author
     * @Description 格式化钉钉文本内容
     * @Date 2021-09-29 18:04
     * @Param
     * @return
     **/
    private String getMarkdown(String title, String msg) 

        if (StringUtils.isEmpty(msg)) 
            return null;
        
        String resultStr = "";

        //日志格式logback配置<pattern>%dyyyy-MM-dd HH:mm:ss.SSS|-|$appName|-|%thread|-|%level|-|%logger|-|%msg%n</pattern>
        String[] msgs = msg.split("\\\\|-\\\\|");
        if (msgs.length > 0) 

            String prefixEnd = "</font> \\n";
            String prefixStart = "####";
            resultStr = "### " + title + prefixEnd +
                    prefixStart + " 应用名：<font color=#1A73E8 size=2>" + getServerName() + prefixEnd +
                    prefixStart + " 类路径：<font color=#666666 size=2>" + msgs[4] + prefixEnd +
                    prefixStart + " 异常时间：<font color=#666666 size=2>" + msgs[0] + prefixEnd +
                    prefixStart + " 异常信息：<font color=#FF0000 size=2>" + msgs[5] + prefixEnd;
        


        return resultStr;
    

2.配置logback.xml

    <appender name="LogDingDing" class="com.logtest.appender.LogDingDingAppender">
        <serverName>log-test</serverName>
        <pattern>%dyyyy-MM-dd HH:mm:ss.SSS|-|$appName|-|%thread|-|%level|-|%logger|-|%msg%n</pattern>
        <!--钉钉机器人Webhook url-->
        <dingdingUrl>*****</dingdingUrl>
        <!--机器人签名密钥 新版需要签名推送-->
        <secret></secret>
        <!--排除的异常类,用逗号分隔-->
        <excludeThrowableClasses></excludeThrowableClasses>
        <!--机器人的推送级别-->
        <level>ERROR</level>
    </appender>

    <root level="INFO">
        <!--<root level="DEBUG">-->
        <appender-ref ref="STDOUT"/>
        <appender-ref ref="infoAppender"/>
        <appender-ref ref="debugAppender"/>
        <appender-ref ref="warnAppender"/>
        <appender-ref ref="errorAppender"/>
        <!--机器人的推送-->
        <appender-ref ref="LogDingDing"/>

    </root>

3.项目所需POM文件

    <dependencies>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-logging</artifactId>
        </dependency>

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.5</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.58</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-autoconfigure</artifactId>
            <version>2.0.1.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.16</version>
        </dependency>
    </dependencies>

4.新建SpringBoot项目&测试

package com.logtest;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import lombok.extern.slf4j.Slf4j;

@SpringBootApplication
@Slf4j
public class LogTestApplication 

    public static void main(String[] args) 
        SpringApplication.run(LogTestApplication.class, args);

        log.error("测试钉钉日志告警");

    

测试结果如下:

5.项目源码

链接：百度网盘 请输入提取码

提取码：b293