JVM 内存大对象监控和优化实践

-Xms4096M -Xmx4096M -Xmn1024M -XX:MetaspaceSize=256M -Djava.security.egd=file:/dev/./urandom -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/data/{runuser}/logs/other

-Xms4096M -Xmx4096M -Xmn1536M -XX:MetaspaceSize=256M -XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -Djava.security.egd=file:/dev/./urandom -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/data/{runuser}/logs/other

Thread 5612: (state = IN_JAVA)
 - org.apache.dubbo.remoting.exchange.codec.ExchangeCodec.encodeResponse(org.apache.dubbo.remoting.Channel, org.apache.dubbo.remoting.buffer.ChannelBuffer, org.apache.dubbo.remoting.exchange.Response) @bci=11, line=282 (Compiled frame; information may be imprecise)
 - org.apache.dubbo.remoting.exchange.codec.ExchangeCodec.encode(org.apache.dubbo.remoting.Channel, org.apache.dubbo.remoting.buffer.ChannelBuffer, java.lang.Object) @bci=34, line=73 (Compiled frame)
 - org.apache.dubbo.rpc.protocol.dubbo.DubboCountCodec.encode(org.apache.dubbo.remoting.Channel, org.apache.dubbo.remoting.buffer.ChannelBuffer, java.lang.Object) @bci=7, line=40 (Compiled frame)
 - org.apache.dubbo.remoting.transport.netty4.NettyCodecAdapter$InternalEncoder.encode(io.netty.channel.ChannelHandlerContext, java.lang.Object, io.netty.buffer.ByteBuf) @bci=51, line=69 (Compiled frame)
 - io.netty.handler.codec.MessageToByteEncoder.write(io.netty.channel.ChannelHandlerContext, java.lang.Object, io.netty.channel.ChannelPromise) @bci=33, line=107 (Compiled frame)
 - io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(java.lang.Object, io.netty.channel.ChannelPromise) @bci=10, line=717 (Compiled frame)
 - io.netty.channel.AbstractChannelHandlerContext.invokeWrite(java.lang.Object, io.netty.channel.ChannelPromise) @bci=10, line=709 (Compiled frame)
...

//org.apache.dubbo.remoting.exchange.codec.ExchangeCodec#encodeResponse
protected void encodeResponse(Channel channel, ChannelBuffer buffer, Response res) throws IOException {
    //...省略部分代码
    try {

        //1、检查响应大小是否超过 payload，如果超过，则抛出ExceedPayloadLimitException异常
        checkPayload(channel, len);
    } catch (Throwable t) {
        //2、重置buffer
        buffer.writerIndex(savedWriteIndex);
        //3、捕获异常后，生成一个新的空响应
        Response r = new Response(res.getId(), res.getVersion());
        r.setStatus(Response.BAD_RESPONSE);        
        //4、ExceedPayloadLimitException异常，将生成的空响应重新发送一遍
        if (t instanceof ExceedPayloadLimitException) {
            r.setErrorMessage(t.getMessage());
            channel.send(r);
            return;
        }         
    }
}
//org.apache.dubbo.remoting.transport.AbstractCodec#checkPayload
protected static void checkPayload(Channel channel, long size) throws IOException {
    int payload = getPayload(channel);
    boolean overPayload = isOverPayload(payload, size);
    if (overPayload) {
        ExceedPayloadLimitException e = new ExceedPayloadLimitException("Data length too large: " + size + ", max payload: " + payload + ", channel: " + channel);
        logger.error(e);
        throw e;
    }
}

/**
 * 自定义dubbo编码类
 **/
public class MusicDubboCountCodec implements Codec2 {
    /**
     * 异常响应池：缓存超过payload大小的responseId
     */
    private static Cache<Long, String> EXCEED_PAYLOAD_LIMIT_CACHE = Caffeine.newBuilder()
        // 缓存总条数
        .maximumSize(100)
        // 过期时间
        .expireAfterWrite(300, TimeUnit.SECONDS)
        // 将value设置为软引用，在OOM前直接淘汰
        .softValues()
        .build();
    @Override
    public void encode(Channel channel, ChannelBuffer buffer, Object message) throws IOException {
        //1、记录数据编码前的buffer位置
        int writeBefore = null == buffer ? 0 : buffer.writerIndex();
        //2、调用原始的编码方法
        dubboCountCodec.encode(channel, buffer, message);
        //3、检查&记录超过payload的信息
        checkOverPayload(message);
        //4、计算对象长度
        int writeAfter = null == buffer ? 0 : buffer.writerIndex();    
        int length = writeAfter - writeBefore;
        //5、超过告警阈值，进行日志打印处理
        warningLengthTooLong(length, message);
    }
    //校验response是否超过payload，超过了，缓存id
    private void checkOverPayload(Object message){
        if(!(message instanceof Response)){
            return;
        }
        Response response = (Response) message;
        //3.1、新的发送过程：通过状态码BAD_RESPONSE与错误信息识别出空响应，并记录响应id
        if(Response.BAD_RESPONSE == response.getStatus() && StrUtil.contains(response.getErrorMessage(), OVER_PAYLOAD_ERROR_MESSAGE)){          
            EXCEED_PAYLOAD_LIMIT_CACHE.put(response.getId(), response.getErrorMessage());
            return;
        }
        //3.2、原先的发送过程：通过异常池识别出超过payload的响应，打印有用的信息
        if(Response.OK == response.getStatus() &&  EXCEED_PAYLOAD_LIMIT_CACHE.getIfPresent(response.getId()) != null){      
            String responseMessage = getResponseMessage(response);
            log.warn("dubbo序列化对象大小超过payload,errorMsg is {},response is {}", EXCEED_PAYLOAD_LIMIT_CACHE.getIfPresent(response.getId()),responseMessage);
        }
    }     
}