springboot整合logback，自定义日志格式，输入es(elk集成)_springboot日志输出到es

记录一次，springboot集成logback，自定义日志格式，输入到es

1.涉及依赖版本

<!--springboot版本-->
<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.1.10.RELEASE</version>
		<relativePath/> 
</parent>
<!--logstash-->
		<dependency>
			<groupId>net.logstash.logback</groupId>
			<artifactId>logstash-logback-encoder</artifactId>
			<version>5.3</version>
		</dependency>

注意事项：springboot版本使用2.1,不要使用最新版本的logstash依赖，过高版本，会出现启动报错，测试过5.3以及4.11版本，均可以启动正常，项目使用5.3

2.logback-spring.xml配置文件

<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
    <!--自定义日志输出参数-->
    <!--<conversionRule conversionWord="msg" converterClass="com.herenit.pharmacycloud.log.LogMessageConverter" />-->
    <!-- 控制台输出 -->
    <springProperty scope="context" name="logstash-server" source="spring.logstash.server"/>
    <springProperty scope="context" name="logstash-port" source="spring.logstash.port"/>
    <springProperty scope="context" name="logstash-service-id" source="spring.logstash.serviceId"/>
 
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>info</level>
        </filter>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder" charset="utf8">
            <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %highlight(%-5level) --- [%15.15(%thread)] %cyan(%-40.40(%logger{40})) : %msg%n</pattern>
        </encoder>
    </appender>
    <!-- 按照每天生成日志文件 -->
   <!-- <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            &lt;!&ndash;日志文件输出的文件名&ndash;&gt;
            <FileNamePattern>${LOG_HOME}/TestWeb.log.%d{yyyy-MM-dd}.log</FileNamePattern>
            &lt;!&ndash;日志文件保留天数&ndash;&gt;
            <MaxHistory>30</MaxHistory>
        </rollingPolicy>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            &lt;!&ndash;格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符&ndash;&gt;
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}  - %msg%n</pattern>
        </encoder>
        &lt;!&ndash;日志文件最大的大小&ndash;&gt;
        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
            <MaxFileSize>10MB</MaxFileSize>
        </triggeringPolicy>
    </appender>-->
 
    <!--输出到logstash的appender-->
    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <param name="Encoding" value="UTF-8"/>
        <remoteHost>${logstash-server}</remoteHost>
        <port>${logstash-port}</port>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder" >
            <customFields>{"serviceId":"${logstash-service-id}"}</customFields>
            <includeMdcKeyName>startTime</includeMdcKeyName>
            <includeMdcKeyName>endTime</includeMdcKeyName>
            <includeMdcKeyName>requestRawJson</includeMdcKeyName>
            <includeMdcKeyName>responseRawJson</includeMdcKeyName>
            <includeMdcKeyName>responseTime</includeMdcKeyName>
            <includeMdcKeyName>url</includeMdcKeyName>
            <includeMdcKeyName>method</includeMdcKeyName>
            <includeMdcKeyName>path</includeMdcKeyName>
        </encoder>
 
    </appender>
    <!--配置异步日志输出-->
    <appender name="LOGSTASH-ASYN" class="ch.qos.logback.classic.AsyncAppender">
        <appender-ref ref="LOGSTASH"/>
    </appender>
 
    <!-- 日志输出级别 -->
    <root level="INFO">
        <appender-ref ref="STDOUT" />
        <appender-ref ref="LOGSTASH-ASYN" />
        <!--<appender-ref ref="FILE" />-->
    </root>
 
 
 
</configuration>

application.yml

spring:
#应用名称
  application:
    name: test
  #日志配置
  logstash:
      server: 127.0.0.1
      port: 5506
      serviceId: service-test-dev

说明:serviceId值，用于es创建对应的项目索引库，具体配置参考下方(logstash对应配置文件)

input{
  tcp{
        mode => "server"
        host => "127.0.0.1"
        port => 5506
        codec => json_lines
    }
}
output{
     elasticsearch{
     hosts =>["127.0.0.1:9200"]
     index=>"%{serviceId}-%{+YYYY.MM.dd}.log"
}
}

3.统一日志拦截(自定义日志输出字段）

@Aspect
@Order(1)
@Component
public class WebLogAspect {
 
 
    private Logger logger = LoggerFactory.getLogger(this.getClass());
 
    ThreadLocal<LogTemplate> logJsonModel = new ThreadLocal<>();
 
 
    //定义项目请求拦截路径，一般是controller层
    @Pointcut("execution(public * com.pp.test.facade..*.*(..))")
    public void webLog() {
    }
 
 
 
    @Before("webLog()")
    public void doBefore(JoinPoint joinPoint) throws Throwable {
        //封装请求入参日志对象
        LogTemplate logTemplate = new LogTemplate();
        logTemplate.setStartTime(LocalDateTime.now());
        logTemplate.setResponseTime(System.currentTimeMillis());
        // 接收到请求，记录请求内容
        ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
        HttpServletRequest request = attributes.getRequest();
        // 记录下请求内容
        logTemplate.setUrl(request.getRequestURL().toString());
        //处理文件上传日志记录
        Object[] args = joinPoint.getArgs();
        Stream<?> stream = ArrayUtils.isEmpty(args) ? Stream.empty() : Arrays.stream(args);
        List<Object> logArgs = stream
                .filter(arg -> (!(arg instanceof HttpServletRequest) && !(arg instanceof HttpServletResponse)))
                .collect(Collectors.toList());
        //过滤后序列化无异常
        String methodName = joinPoint.getSignature().getName();
              logTemplate.setRequestRawJson(StringEscapeUtils.unescapeJavaScript(JSONObject.toJSONString(JSON.toJSONString(logArgs))));
            logTemplate.setMethod(methodName);
            logTemplate.setPath(joinPoint.getSignature().getDeclaringTypeName());
       
        logJsonModel.set(logTemplate);
    }
 
    /**
     * @Description:记录响应请求正常日志
     * @Param: [ret]
     * @Return: void
     */
    @AfterReturning(returning = "ret", pointcut = "webLog()")
    public void doAfterReturning(JoinPoint joinPoint,Object ret) throws Throwable {
        //过滤后序列化无异常
        String methodName = joinPoint.getSignature().getName();
      logJsonModel.get().setResponseRawJson(StringEscapeUtils.unescapeJavaScript(JSONObject.toJSONString(ret)));
            logJsonModel.get().setResponseTime(System.currentTimeMillis() - logJsonModel.get().getResponseTime());
            logJsonModel.get().setEndTime(LocalDateTime.now());
            addLogData();
            logger.info(logJsonModel.get().toString());
        
    }
 
//自定义日志需要输入的字段
    private void addLogData() {
            MDC.put("startTime", logJsonModel.get().getStartTime().format(DateTimeFormatter.ofPattern(DateTimeUtils.DATE_TIME)));
            MDC.put("endTime", logJsonModel.get().getEndTime().format(DateTimeFormatter.ofPattern(DateTimeUtils.DATE_TIME)));
            MDC.put("requestRawJson", logJsonModel.get().getRequestRawJson());
            MDC.put("responseRawJson",logJsonModel.get().getResponseRawJson());
            MDC.put("responseTime", Long.toString(logJsonModel.get().getResponseTime()));
            MDC.put("url", logJsonModel.get().getUrl());
            MDC.put("method", logJsonModel.get().getMethod());
            MDC.put("path", logJsonModel.get().getPath());
    }
 
    /**
     * @Description:记录响应请求异常日志
     * @Param: [ex]
     * @Return: void
     */
    @AfterThrowing(throwing="ex",pointcut = "webLog()")
    public void afterThrowing(Exception  ex) {
        String errorCode = "";
        String msg = "";
        if (ex instanceof BusinessException) {
            BusinessException be = (BusinessException)ex;
            errorCode = be.getCode();
            msg = be.getMessage();
        } else if (ex instanceof MethodArgumentNotValidException) {
            MethodArgumentNotValidException be = (MethodArgumentNotValidException)ex;
            msg = be.getBindingResult().getAllErrors().stream()
                    .map(DefaultMessageSourceResolvable::getDefaultMessage)
                    .reduce((m1, m2) ->  m1 + ";" + m2)
                    .orElse("未获取到错误信息");
            errorCode = RespCode.FAILED.getCode();
        }else {
            errorCode = RespCode.FAILED.getCode();
            msg = ex.getMessage();
        }
//        logger.error("异常code:"+errorCode+",异常信息："+msg, e);
        JSONObject repJson=new JSONObject();
        repJson.put("errorCode",errorCode);
        repJson.put("errorMsg",msg);
        logJsonModel.get().setResponseRawJson(StringEscapeUtils.unescapeJavaScript(JSONObject.toJSONString(repJson)));
        logJsonModel.get().setResponseTime(System.currentTimeMillis() - logJsonModel.get().getResponseTime());
        logJsonModel.get().setEndTime(LocalDateTime.now());
        logJsonModel.get().setMessage(msg);
        addLogData();
//        logger.error("异常code:"+errorCode+",异常信息："+msg, ex);
        logger.info(logJsonModel.get().toString());
    }
}

4.问题以及总结

针对自定义字段输入到日志格式中，想过重写相关的底层代码，也测试过，但是结果都不是本人想要的效果，

net.logstash.logback.encoder.LogstashEncoder，通过重写改实现类，但是改类提供更改的方法有限，

希望有大佬，针对这个问题提供宝贵经验。

另外还存在一个问题，springboot启动的所有日志，也会一条一条输入到es中，这个就很烦，目前使用比较笨的方法，

写了一个拦截器，过滤springboot项目的无关的启动日志；

也希望针对这个问题，大佬们提供有效可行的建议