配置Spring项目输出JSON到LogStash或者使用FileBeat收集上传到ELK

Posted 2020-11-27

一、使用LogStash

在项目中添加Gradle依赖，然后Sync项目：

"net.logstash.logback:logstash-logback-encoder:4.11",

或者使用Maven：

<!-- https://mvnrepository.com/artifact/net.logstash.logback/logstash-logback-encoder -->
<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>4.11</version>
</dependency>

然后添加配置文件：

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
  <contextName>${HOSTNAME}</contextName>
  <property name="LOG_PATH" value="/var/log" />
  <springProperty scope="context" name="appName" source="spring.application.name" />
  <springProperty scope="context" name="ip" source="spring.cloud.client.ipAddress" />
  <property name="CONSOLE_LOG_PATTERN"
            value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n"/>

  <appender name="FILEERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-error.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>error</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="FILEWARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-warn.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-warn-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>warn</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="FILEINFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-info.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-info-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>

    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>info</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="logstash" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}.json</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>${LOG_PATH}/${appName}/${appName}-%d{yyyy-MM-dd}.json</fileNamePattern>
      <maxHistory>7</maxHistory>
    </rollingPolicy>
    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
      <providers>
        <timestamp>
          <timeZone>UTC</timeZone>
        </timestamp>
        <pattern>
          <pattern>
            {
            "ip": "${ip}",
            "app": "${appName}",
            "level": "%level",
            "trace": "%X{X-B3-TraceId:-}",
            "span": "%X{X-B3-SpanId:-}",
            "parent": "%X{X-B3-ParentSpanId:-}",
            "thread": "%thread",
            "class": "%logger{40}",
            "message": "%message",
            "stack_trace": "%exception{10}"
            }
          </pattern>
        </pattern>
      </providers>
    </encoder>
  </appender>

  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
    <encoder>
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
      <level>debug</level>
    </filter>
  </appender>

  <logger name="org.springframework" level="INFO" />
  <logger name="org.hibernate" level="INFO" />
  <logger name="com.kingboy.repository" level="DEBUG" />

  <root level="INFO">
    <appender-ref ref="FILEERROR" />
    <appender-ref ref="FILEWARN" />
    <appender-ref ref="FILEINFO" />
    <appender-ref ref="logstash" />
    <appender-ref ref="STDOUT" />
  </root>
</configuration>

在上面的配置文件中：

• LOG_PATH主要配置日志输出目录，在后面使用LOG_PATH进行路径拼接
• 两个springProperty用于配置使用TCP Socket方式读取日志的IPv4地址和Port号
• CONSOLE_LOG_PATTERN用于配置日志格式
• FILEERROR用于配置日志器实现滚动输出
• rollingPolicy的XML配置块指定了如何切分文件的策略

LogStash需要安装一个插件：

bin/plugin install logstash-codec-json_lines

参考：https://github.com/logstash/logstash-logback-encoder

二、使用FileBeat

有些公司的日志架构可能是通过各种Beats收集然后通过LogStash过滤后写入Kafka再写入ElasticSearch中的，因此不需要整合logstash-logback-endcoder，那么你只需要整合一个配置类即可：

@Configuration
public class WebApplicationConfig extends WebMvcConfigurerAdapter {

    @Override
    public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
        super.configureMessageConverters(converters);
        // 1. 需要定义一个convert转换消息的对象;
        FastJsonHttpMessageConverter fastJsonHttpMessageConverter = new FastJsonHttpMessageConverter();
        // 2. 添加FastJson的配置信息，比如：是否要格式化返回的json数据;
        FastJsonConfig fastJsonConfig = new FastJsonConfig();
        fastJsonConfig.setSerializerFeatures(SerializerFeature.PrettyFormat,
                                      SerializerFeature.WriteMapNullValue,
                                      SerializerFeature.WriteNullStringAsEmpty,
                                      SerializerFeature.DisableCircularReferenceDetect,
                                      SerializerFeature.WriteNullListAsEmpty);
        // 3. 处理中文乱码问题
        List<MediaType> fastMediaTypes = new ArrayList<>();
        fastMediaTypes.add(MediaType.APPLICATION_JSON_UTF8);
        // 4. 在convert中添加配置信息.
        fastJsonHttpMessageConverter.setSupportedMediaTypes(fastMediaTypes);
        fastJsonHttpMessageConverter.setFastJsonConfig(fastJsonConfig);
        // 5. 将convert添加到converters当中.
        converters.add(fastJsonHttpMessageConverter);
    }
}