javascript 简单的NodeJS脚本，用于抓取网站并获取网址

Posted 2021-05-03

var Crawler = require("js-crawler");
var url = require('url');

if (process.argv.length <= 2) {
    console.log("Usage: " + __filename + " http://www.yahoo.com");
    process.exit(-1);
}
 
var crawler = new Crawler().configure({
  maxRequestsPerSecond: 10,
  maxConcurrentRequests: 10,
  depth: 99,
});

var url1 = process.argv[2];
var hostname = (url.parse(url1).hostname);

var parts = hostname.split('.');
var subdomain = parts.shift();
var upperleveldomain = parts.join('.');

crawler.crawl({
  url: url1,
  success: function(page) {
    if((url.parse(page.url).hostname).indexOf(upperleveldomain) > -1) {
    	console.log(page.url);
     };
  },
});