JAVA爬数据

Posted 2021-03-02 zhetiankj

/**
 * httClient 请求 GET
 * 获取数据测试
 */
@Test
public void testDepartmentList() {
    //数据链接
    String url = "https://search-x.jd.com/Search?callback=jQuery1352084&area=22&enc=utf-8&keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&adType=7&urlcid3=672&page=1&ad_ids=291%3A33&xtest=new_search&_=1585561488653";
    //网页链接
    String referer = "https://search.jd.com/search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&enc=utf-8&qrst=1&rt=1&stop=1&spm=2.1.1&vt=2&cid2=671&cid3=672&ev=exbrand_%E8%81%94%E6%83%B3%EF%BC%88Lenovo%EF%BC%89%5E&uc=0";
    System.out.println("请求url:" + url);
    HttpRequest request = HttpUtil.createGet(url);
    request.header("Referer", referer);
    String str2 = request.execute().body();

    int length = str2.length();
    int i = str2.indexOf("(");
    System.out.println(i);
    str2 = str2.substring(i+1, length - 1);
    System.out.println("请求响应:" + str2);

    JSONObject jsonObject = JSON.parseObject(str2);
    System.out.println(jsonObject);
    JSONArray jsonArray = jsonObject.getJSONArray("291");
    int size = jsonArray.size();
    for (int j = 0; j <size ; j++) {
        JSONObject jsonObject1 = jsonArray.getJSONObject(j);
        String ad_title = jsonObject1.getString("ad_title");
        
        System.out.println(j+"ad_title="+ad_title);
    }

    System.out.println("请求响应:" + str2);
}