如何用JAVA爬取AJAX加载后的页面

Posted 2023-03-30

普通的爬取是抓不了js的之后的数据的 可以用phantomjs或者htmlUnit实现
附上phantomjs示列代码
package cn.wang.utils;

import java.util.Random;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.CookieManager;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;

public class htmlUnitUtils

static WebClient webClient = null;

static Random random = new Random();

static
//1.创建对象
webClient = new WebClient(BrowserVersion.CHROME);
//2.设置参数
//启动js
webClient.getOptions().setjavascriptEnabled(true);
//关闭css渲染
webClient.getOptions().setCssEnabled(false);
//启动重定向
webClient.getOptions().setRedirectEnabled(true);
//设置连接超时时间 ，这里是10S。如果为0，则无限期等待
webClient.getOptions().setTimeout(1000 * 15);
//启动cookie管理
webClient.setCookieManager(new CookieManager());
//启动ajax代理
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
//js运行时错误，是否抛出异常
webClient.getOptions().setThrowExceptionOnScriptError(false);
//设置浏览器请求信息
webClient.addRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
webClient.addRequestHeader("Accept-Encoding", "gzip, deflate");
webClient.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
webClient.addRequestHeader("Connection", "keep-alive");
webClient.addRequestHeader("Upgrade-Insecure-Requests", "1");


public static void runJs(String url)
try
webClient.addRequestHeader("User-Agent", Constant.useragents[random.nextInt(Constant.useragents.length)]);
//等待js渲染执行 waitime等待时间(ms)
webClient.waitForBackgroundJavaScript(1000 * 10);
//3.获取页面
webClient.getPage(url);
catch (Exception e)
e.printStackTrace();
finally
if(webClient != null)
webClient.close();




public static void main(String[] args)
runJs("http://www.gou.hk/");
System.setProperty("phantomjs.binary.path", "D:\\works\\tool\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe");


参考技术A 分情况
一般简单的网页通过get参数进行分页 这种情况就通过构造url来进行分页
有些网站是通过post参数来进行分页，那就用代码post的相应的参数给网站
比较复杂的ajax的分页需要通过抓包来实现