javascript 使用phantomjs和horseman抓取并抓取包含ajax内容的网页

Posted 2021-05-03

var Horseman = require('node-horseman');
var horseman = new Horseman({phantomPath:'C:\\Users\\Marketing-1_2\\phantomjs-2.1.1-windows\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe'});
var visited = [];
var list = [];
function gotopage(url){
horseman  
  .open(url)
  evaluate( function(selector){
         var urlsInPage = [];
		 $( selector ).each(function(){
		  urlsInPage.push($(this).attr('href'));
		       });
	     return urlsInPage;
      
    }, 'a')
  .then(function(output){
    for (var i = Auction.auctions.length - 1; i >= 0; i--) {
		 for (var j =0;j<visited.length;j++){
			 if (visited[j] == output[i]){
				 output.splice(i,1);
				 break;
		     }
	     }
	}
  })
  .evaluate( function(selector){
      return   $( selector ).html();
      
    }, '.hd h3')
  .then(function(output){
    console.log(output);
  })
  .evaluate( function(selector){
      return   $( selector ).html();
      
    }, '.hd h3')
  .then(function(output){
    console.log(output);
  })
  .evaluate( function(selector){
      return   $( selector ).html();
      
    }, 'title')
  .then(function(output){
    console.log(output);
  })
  .close();
}