php curl采集，服务器gzip压缩返回数据怎么办

Posted 2023-05-07

参考技术A function gzip_get_contents($url)

$ch=curl_init($url);
curl_setopt($ch, CURLOPT_ENCODING. 'gzip');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$content=curl_exec($ch);
curl_close($ch);
return $content;

基于Filter实现Gzip数据压缩

在web开发中，当服务器端向客户端返回的数据量比较大时，我们可以通过Gzip对数据进行压缩处理

注意：如果小数据量进行压缩，压缩后的数据可能比原始数据还大；所以response返回数据量比较小时不推荐使用

 

这里代码结构，红叉不需要关注，这里maven环境的问题，不影响我们的主线

 

web.xml

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xmlns="http://java.sun.com/xml/ns/javaee" 
    xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" 
    id="WebApp_ID" version="3.0">
  <display-name>gzipfilter</display-name>
<filter>
     <filter-name>gzip</filter-name>
     <filter-class>gzipfilter.GzipFilter</filter-class>
  </filter>
  <filter-mapping>
     <filter-name>gzip</filter-name>
     <url-pattern>/servlet/*</url-pattern>
  </filter-mapping>
  <servlet>
    <servlet-name>TestServlet</servlet-name>
    <servlet-class>gzipfilter.TestServlet</servlet-class>
  </servlet>

  <servlet-mapping>
    <servlet-name>TestServlet</servlet-name>
    <url-pattern>/servlet/test</url-pattern>
  </servlet-mapping>    
  <welcome-file-list>
    <welcome-file>index.jsp</welcome-file>
  </welcome-file-list>
</web-app>

 

pom.xml

    <dependencies>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>jstl</artifactId>
            <version>1.2</version>
            <type>jar</type>
        </dependency>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
            <version>2.5</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>javax.servlet.jsp</groupId>
            <artifactId>jsp-api</artifactId>
            <version>2.1</version>
            <scope>provided</scope>
        </dependency>
    </dependencies>

 

GzipFilter

public class GzipFilter implements Filter{

    public void init(FilterConfig config) throws ServletException {
        
    }

    public void destroy() {
        
    }

    public void doFilter(ServletRequest req, ServletResponse res,
            FilterChain chain) throws IOException, ServletException {
        HttpServletRequest request = (HttpServletRequest) req;
        HttpServletResponse response = (HttpServletResponse) res;
        ResponseWrapper responseWrapper = new ResponseWrapper(response);
        
        chain.doFilter(request, responseWrapper);//通过过滤器  把响应数据写进自己的response中
        
        //获得响应数据
        ByteArrayOutputStream buffer=responseWrapper.getBuffer();
        byte[] bytes=buffer.toByteArray();
        
        //压缩
        ByteArrayOutputStream baout=new ByteArrayOutputStream();
        GZIPOutputStream gout=new GZIPOutputStream(baout);
        gout.write(bytes);
        gout.close();//拿到压缩后的数据----响应前端
        byte[] gzipbytes = baout.toByteArray();
        response.setHeader("Content-Encoding", "gzip");
        System.out.println(gzipbytes.length+"-------------压缩后");
        response.getOutputStream().write(gzipbytes);
        
    }
}
/**
 * 
 * 对原始response进行包装
 *
 */
class ResponseWrapper extends HttpServletResponseWrapper {
    
    private ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    private PrintWriter pw;

    public ResponseWrapper(HttpServletResponse response) {
        super(response);
    }
     
    @Override
    public ServletOutputStream getOutputStream() throws IOException {
        OutputStreamWrapper out = new OutputStreamWrapper(buffer);
        return out;
    }
    
    @Override
    public PrintWriter getWriter() throws IOException {
        if(pw==null){
            OutputStreamWrapper out = new OutputStreamWrapper(buffer);
            pw = new PrintWriter(new OutputStreamWriter(out, "UTF-8"));
        }
        return pw;
    }
    
    public ByteArrayOutputStream getBuffer(){
        return buffer;
    }
}

/**
 * 
 * 包装 outputStream
 *
 */
class OutputStreamWrapper extends ServletOutputStream{
    
    private ByteArrayOutputStream buffer;
    
    public OutputStreamWrapper(ByteArrayOutputStream buffer) {
         super();
         this.buffer=buffer;
    }

    @Override
    public void write(int b) throws IOException {
        this.buffer.write(b);
    }
    
}

 

TestServlet

public class TestServlet extends HttpServlet {
     
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        String name="a";
        for(int i=0;i<10;i++){
            name=name+name;
        }
        System.out.println(name.getBytes().length+"==============压缩前");
        response.getOutputStream().write(name.getBytes());
    }

     
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        doGet(request, response);
    }
 
}

 

index.jsp

<%@ page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<%
String path = request.getContextPath();
String basePath = request.getScheme()+"://"+request.getServerName()+":"+request.getServerPort()+path+"/";
%>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<base href="<%=basePath%>">
<title>My JSP ‘index.jsp‘ starting page</title>
<meta http-equiv="pragma" content="no-cache">
<meta http-equiv="cache-control" content="no-cache">
</head>
<body>
  <a href="servlet/test">点点</a>
</body>
</html>

 

结果：