JAVA——微博热搜爬虫

Posted 2021-09-11 Starzkg

源代码：https://gitee.com/my-zstu/myzstu/blob/develop/myzstu-spyder/src/main/java/club/zstuca/myzstu/spyder/misc/WeiboTopSpyder.java

解决方案

package club.zstuca.myzstu.spyder.misc;

import club.zstuca.myzstu.spyder.misc.entity.WeiboTopItem;
import club.zstuca.myzstu.utils.http.HttpUtil;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;

public class WeiboTopSpyder {
    public String url = "https://s.weibo.com/top/summary";
    public List<WeiboTopItem> run(){
        try {
            HttpResponse response = HttpUtil.doGet(url);
            String content = EntityUtils.toString(response.getEntity());
            if (content == null || content.isEmpty()) {
                return null;
            }
            Document document = Jsoup.parse(content);
            Elements elements = document.select("#pl_top_realtimehot").select("tbody").select("tr");
            List<WeiboTopItem> list  = new ArrayList<>();
            elements.forEach(item->{
                WeiboTopItem topItem = new WeiboTopItem();
                Element element = item.select("td").get(1).select("a").first();
                if (element == null) {
                    return;
                }
                topItem.setName(element.text());
                topItem.setUrl("https://s.weibo.com" + element.attr("href"));
                list.add(topItem);
            });
            return list;
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }
}