IngCrawler

Posted 2020-08-28 jieyuefeng

import java.io.IOException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509TrustManager;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

/*
CREATE TABLE `ing` (
  `id` int(11) unsigned NOT NULL,
  `url` varchar(500) DEFAULT NULL,
  `user` varchar(100) DEFAULT NULL,
  `date` varchar(30) DEFAULT NULL,
  `content` varchar(5000) DEFAULT NULL,
  `lucky` tinyint(4) DEFAULT NULL,
  `userlink` varchar(500) DEFAULT NULL,
  `mtime` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `comment` (
  `id` int(11) unsigned NOT NULL,
  `ingid` int(11) DEFAULT NULL,
  `user` varchar(100) DEFAULT NULL,
  `content` varchar(5000) DEFAULT NULL,
  `date` varchar(30) DEFAULT NULL,
  `userlink` varchar(100) DEFAULT NULL,
  `mtime` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 * */
public class IngCrawler {
    static {
        try {
            HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
                public boolean verify(String hostname, SSLSession session) {
                    return true;
                }
            });

            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, new X509TrustManager[] { new X509TrustManager() {
                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[0];
                }
            } }, new SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws IOException {
        int id = Inserter.getNextId();
        int lastestid = Crawler.getLastestId();

        for (; id <= lastestid; id++) {
            Crawler.crawl("https://ing.cnblogs.com/u/1/status/" + id, id);
        }
    }

    static class Ing {
        int id;
        String url;
        String user;
        String date;
        String content;
        boolean lucky;
        String userlink;
        List<Comment> comments = new ArrayList<Comment>();

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder(id + " - [" + date + "][" + user + "] - " + content);
            for (Comment c : this.comments) {
                sb.append("\\n\\t" + c);
            }
            return sb.toString();
        }

        static Ing parseIng(Document doc, String url, int id) {
            Ing ing = new Ing();
            ing.id = id;
            ing.url = url;

            if (doc.select(".ing_detail_title").size() == 0) {
                return ing;
            }

            ing.user = doc.select(".ing_item_author").text().trim();
            ing.userlink = doc.select(".ing_item_author").attr("href");
            ing.date = doc.select(".ing_detail_title").text().trim();
            if (ing.date.indexOf("：") != -1) {
                ing.date = ing.date.substring(ing.date.indexOf("：") + 1).trim();
            }
            ing.content = doc.select("#ing_detail_body").text().trim();
            ing.lucky = doc.select(".ing_icon_lucky").size() > 0;
            for (Element e : doc.select("#comment_block_" + id).get(0).children()) {
                ing.comments.add(Comment.parseComment(e, id));
            }
            return ing;
        }

        static class Comment {
            int id;
            int ingid;
            String user;
            String content;
            String date;
            String userlink;

            static Comment parseComment(Element e, int ingid) {
                Comment comment = new Comment();
                comment.id = Integer.parseInt(e.id().substring(8));
                comment.ingid = ingid;

                comment.user = e.select("#comment_author_" + comment.id).text().trim();
                comment.userlink = e.select("#comment_author_" + comment.id).attr("href");
                comment.date = e.select(".text_green").attr("title").trim();

                e.select("#comment_author_" + comment.id).remove();
                e.select(".text_green").remove();
                e.select(".gray3").remove();

                comment.content = e.select("div").text().trim();
                if (comment.content.startsWith(":")) {
                    comment.content = comment.content.substring(1).trim();
                }

                return comment;
            }

            @Override
            public String toString() {
                return "[" + user + "] - " + content;
            }
        }

    }

    static class Crawler implements Runnable {
        static ExecutorService crawler = Executors.newFixedThreadPool(10);

        String url;
        int id;

        public Crawler(String url, int id) {
            this.url = url;
            this.id = id;
        }

        public static int getLastestId() {
            return 1054304;
        }

        public static void crawl(String url, int id) {
            crawler.execute(new Crawler(url, id));
        }

        @Override
        public void run() {
            System.out.println("crawl for: " + url);
            try {
                String cookie = "YOUR COOKIE HERE";
                String useragent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/55.0.2883.95 Safari/537.36";
                Inserter.insert(Ing.parseIng(Jsoup.connect(url).header("cookie", cookie).userAgent(useragent).get(), url, id));
            } catch (IOException e) {
                e.printStackTrace();
            }

        }

    }

    static class Inserter implements Runnable {
        static ExecutorService inserter = Executors.newFixedThreadPool(1);

        static Connection conn;
        static PreparedStatement pstating, pstatcmt;

        static {
            try {
                Class.forName("com.mysql.jdbc.Driver");
                conn = DriverManager.getConnection(
                        "jdbc:mysql://localhost:3306/ing?useUnicode=true&characterEncoding=utf-8&autoReconnect=true", "root", "");

                pstating = conn
                        .prepareStatement("insert into ing (id,url,user,date,content,lucky,userlink) values (?,?,?,?,?,?,?)");
                pstatcmt = conn
                        .prepareStatement("insert into comment (id,ingid,user,content,date,userlink) values (?,?,?,?,?,?)");
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        Ing ing;

        public Inserter(Ing ing) {
            this.ing = ing;
        }

        public static int getNextId() {
            try {
                Statement stat = conn.createStatement();
                ResultSet rs = stat.executeQuery("select max(id) as id from ing");
                if (rs.next()) {
                    return rs.getInt("id") + 1;
                }
            } catch (SQLException e) {
                // ignore
            }
            return 1;
        }

        static int no = 0;

        public static void insert(Ing ing) {
            inserter.execute(new Inserter(ing));
        }

        @Override
        public void run() {
            System.out.println(++no + ". " + ing);
            try {
                pstating.setInt(1, ing.id);
                pstating.setString(2, ing.url);
                pstating.setString(3, ing.user);
                pstating.setString(4, ing.date);
                pstating.setString(5, ing.content);
                pstating.setInt(6, ing.lucky ? 1 : 0);
                pstating.setString(7, ing.userlink);
                pstating.executeUpdate();

                for (Ing.Comment c : ing.comments) {
                    pstatcmt.setInt(1, c.id);
                    pstatcmt.setInt(2, c.ingid);
                    pstatcmt.setString(3, c.user);
                    pstatcmt.setString(4, c.content);
                    pstatcmt.setString(5, c.date);
                    pstatcmt.setString(6, c.userlink);
                    pstatcmt.executeUpdate();
                }
            } catch (SQLException e) {
                System.err.println("ERROR - " + e.getMessage() + " - " + ing);
            }
        }

    }

}