Connection Reset

Posted 2020-10-06

异常 Connection Reset

HttpUrlTool.java 中没有设定超时时间,导致目标服务器海闻关服务后,出现大量的链接被占用,却没有被释放.

于是无法再获取新的http链接,最终导致Connection Reset.

 

dump线程转储内容如下:

"[STUCK] ExecuteThread: ‘28‘ for queue: ‘weblogic.kernel.Default (self-tuning)‘" RUNNABLE native
          
                java.net.SocketInputStream.socketRead0(Native Method)
          
                java.net.SocketInputStream.read(SocketInputStream.java:129)
          
                org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)
          
                org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)
          
                org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)
          
                org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
          
                org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
          
                org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
          
                org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)
          
                sun.reflect.GeneratedMethodAccessor6351.invoke(Unknown Source)
          
                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
          
                java.lang.reflect.Method.invoke(Method.java:597)
          
                org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)
          
                com.sun.proxy.$Proxy105.receiveResponseHeader(Unknown Source)
          
                org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
          
                org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
          
                org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:254)
          
                org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)
          
                org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)
          
                org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)
          
                org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)
          
                org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
          
                org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
          
                com.isoftstone.pcis.common.utils.HttpUrlTool.sendSOAPActionInfo(HttpUrlTool.java:558)
          
                com.isoftstone.pcis.common.utils.HttpUrlTool.sendSoapInfoHW(HttpUrlTool.java:600)
          
                com.isoftstone.pcis.billing.service.BillingService.sendBillingInfo(BillingService.java:183)
          
                com.isoftstone.pcis.billing.action.BillingBizAction.toBilling(BillingBizAction.java:290)
          
                sun.reflect.GeneratedMethodAccessor12548.invoke(Unknown Source)
          
                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
          
                java.lang.reflect.Method.invoke(Method.java:597)
          
                org.apache.commons.beanutils.MethodUtils.invokeExactMethod(MethodUtils.java:404)
          
                org.apache.commons.beanutils.MethodUtils.invokeExactMethod(MethodUtils.java:354)
          
                com.isoftstone.fwk.facade.BizControllerImpl.handleRequest(BizControllerImpl.java:126)
          
                com.isoftstone.fwk.facade.BizControllerPojoAdapter.handleRequest(BizControllerPojoAdapter.java:37)
          
                com.isoftstone.fwk.action.BaseWebAction.perform(BaseWebAction.java:67)
          
                sun.reflect.GeneratedMethodAccessor8303.invoke(Unknown Source)
          
                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
          
                java.lang.reflect.Method.invoke(Method.java:597)
          
                org.apache.commons.beanutils.MethodUtils.invokeExactMethod(MethodUtils.java:404)
          
                org.apache.commons.beanutils.MethodUtils.invokeExactMethod(MethodUtils.java:354)
          
                com.isoftstone.fwk.action.ActionControl.excute(ActionControl.java:19)
          
                com.isoftstone.fwk.web.WebControl.doAction(WebControl.java:225)
          
                com.isoftstone.fwk.web.WebControl.doPost(WebControl.java:47)
          
                javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
          
                javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
          
                weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:227)
          
                weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)
          
                weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:301)
          
                weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)
          
                weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:60)
          
                com.isoftstone.fwk.web.filter.DWCallbackFilter.doFilter(DWCallbackFilter.java:110)
          
                weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:60)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:359)
          
                org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:109)
          
                org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:83)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.ui.SessionFixationProtectionFilter.doFilterHttp(SessionFixationProtectionFilter.java:67)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.ui.ExceptionTranslationFilter.doFilterHttp(ExceptionTranslationFilter.java:101)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.wrapper.SecurityContextHolderAwareRequestFilter.doFilterHttp(SecurityContextHolderAwareRequestFilter.java:91)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.ui.AbstractProcessingFilter.doFilterHttp(AbstractProcessingFilter.java:116)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:59)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.context.HttpSessionContextIntegrationFilter.doFilterHttp(HttpSessionContextIntegrationFilter.java:238)
          
                org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
          
                org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
          
                org.springframework.security.util.FilterChainProxy.doFilter(FilterChainProxy.java:174)
          
                org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:236)
          
                org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
          
                weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:60)
          
                org.jasig.cas.client.session.SingleSignOutFilter.doFilter(SingleSignOutFilter.java:99)
          
                weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:60)
          
                com.isoftstone.iaeap.web.filter.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:97)
          
                weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:60)
          
                weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3748)
          
                weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3714)
          
                weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
          
                weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
          
                weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2283)
          
                weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2182)
          
                weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1491)
          
                weblogic.work.ExecuteThread.execute(ExecuteThread.java:263)
          
                weblogic.work.ExecuteThread.run(ExecuteThread.java:221)

 

解决方案

给apache 的HttpClients配置配置超时时间setSocketTimeout(10000) 10秒,10秒后目标服务器如果未响应自动断开连接,认为是ReadTimeoutException,就会正常释放链接了.见以下加粗部分

com.isoftstone.pcis.common.utils.HttpUrlTool.sendSOAPActionInfo(HttpUrlTool.java:558)附近

public static String sendSOAPActionInfo(Map<String,String> httpMap , String requestXml) throws Exception {
        String charSet = httpMap.get("encoding") == null ? HttpUrlTool.UTF_8 : httpMap.get("encoding");
        String name = httpMap.get("name")==null ? "接口" : httpMap.get("name");
        
        CloseableHttpClient client = HttpClients.custom().build();
        HttpPost httpPost = new HttpPost(httpMap.get("url"));

        RequestConfig requestConfig = RequestConfig.custom()
        .setSocketTimeout(10000)
        .setConnectTimeout(5000)
        .setConnectionRequestTimeout(5000)
        .build();
        httpPost.setConfig(requestConfig);
        
        String returnXml = null;
        try{
            StringEntity content = new StringEntity(requestXml.toString(), Charset.forName(charSet));// 第二个参数，设置后才会对，内容进行编码
            content.setContentType("text/xml; charset="+ charSet);
            content.setContentEncoding(charSet);
            httpPost.setEntity(content);
            if(httpMap.get("SOAPAction") != null) 
                httpPost.setHeader("SOAPAction", httpMap.get("SOAPAction"));
            HttpResponse response = client.execute(httpPost);
            int status = response.getStatusLine().getStatusCode();
            // 先从响应头得到实体
            HttpEntity resEntity = response.getEntity();
            if (status == HttpStatus.SC_OK && resEntity != null) {
                String sentity = EntityUtils.toString(resEntity, charSet);
                if(StringTool.isNotBlank(sentity)){
                    returnXml = sentity;
                }else{
                    logger.error("连接 "+name+" 异常,无返回内容！");
                    throw new Exception("无返回内容！");
                }
            } else {
                logger.error("没有正确连接 "+name+" HTTP服务返回有误,返回编码："+ status);
                throw new Exception("没有正确连接 "+name+" HTTP服务返回有误,返回编码："+ status);
            }      
        } catch(Exception e){
            logger.error("连接 "+name+" 异常",e);
            throw new Exception("连接 "+name+" 异常："+e.getMessage(),e);
        } finally{
            if (httpPost!=null) 
                httpPost.releaseConnection();
        }
        return returnXml;
    }