java抓取网页数据

Posted 2022-12-05 zhangtian6691844

java抓取网页数据获取网页中所有的链接实例分享，使用方法，只要实例化htmlParser时传入网页地址就可以了

效果图

复制代码 代码如下:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HtmlParser
 /**
  * 要分析的网页
  */
 String htmlUrl;

 /**
  * 分析结果
  */
 ArrayList<String> hrefList = new ArrayList();

 /**
  * 网页编码方式
  */
 String charSet;

 public HtmlParser(String htmlUrl)
  // TODO 自动生成的构造函数存根
  this.htmlUrl = htmlUrl;
 

 /**
  * 获取分析结果
  *
  * @throws IOException
  */
 public ArrayList<String> getHrefList() throws IOException

  parser();
  return hrefList;
 

 /**
  * 解析网页链接
  *
  * @return
  * @throws IOException
  */
 private void parser() throws IOException
  URL url = new URL(htmlUrl);
  HttpURLConnection connection = (HttpURLConnection) url.openConnection();
  connection.setDoOutput(true);

  String contenttype = connection.getContentType();
  charSet = getCharset(contenttype);

  InputStreamReader isr = new InputStreamReader(
    connection.getInputStream(), charSet);
  BufferedReader br = new BufferedReader(isr);

  String str = null, rs = null;
  while ((str = br.readLine()) != null)
   rs = getHref(str);

   if (rs != null)
    hrefList.add(rs);
  

 

 /**
  * 获取网页编码方式
  *
  * @param str
  */
 private String getCharset(String str)
  Pattern pattern = Pattern.compile("charset=.*");
  Matcher matcher = pattern.matcher(str);
  if (matcher.find())
   return matcher.group(0).split("charset=")[1];
  return null;
 

 /**
  * 从一行字符串中读取链接
  *
  * @return
  */
 private String getHref(String str)
  Pattern pattern = Pattern.compile("<a href=.*</a>");
  Matcher matcher = pattern.matcher(str);
  if (matcher.find())
   return matcher.group(0);
  return null;
 

 public static void main(String[] arg) throws IOException
  HtmlParser a = new HtmlParser("http://news.163.com/");
  ArrayList<String> hrefList = a.getHrefList();
  for (int i = 0; i < hrefList.size(); i++)
   System.out.println(hrefList.get(i));