phantomjs 截取twitter的网页（动态生成的页面）

Posted 2020-08-29 一起来学python

// This example shows how to render pages that perform AJAX calls
// upon page load.
//
// Instead of waiting a fixed amount of time before doing the render,
// we are keeping track of every resource that is loaded.
//
// Once all resources are loaded, we wait a small amount of time
// (resourceWait) in case these resources load other resources.
//
// The page is rendered after a maximum amount of time (maxRenderTime)
// or if no new resources are loaded.

//page.onResourceRequested = function(requestData, networkRequest) {} //page的所要加载的资源在发起请求时，都可以回调该函数

//page.onResourceReceived = function(response) {} //page的所要加载的资源在加载过程中，每加载一个相关资源，都会在此先做出响应，它相当于

var fs = require("fs");
var resourceWait = 300,
maxRenderWait = 10000,
url=‘https://twitter.com/CXA10‘;

var page = require(‘webpage‘).create(),
count = 0,
forcedRenderTimeout,
renderTimeout;

page.viewportSize = { width: 1280, height : 1024 };

function doRender() {
page.render(‘twitter.png‘);
fs.write(‘1.html‘, page.content, ‘w‘);
phantom.exit();
}

page.onResourceRequested = function (req) {
count += 1;
console.log(‘> ‘ + req.id + ‘ - ‘ + req.url);
clearTimeout(renderTimeout);
};

page.onResourceReceived = function (res) {
if (!res.stage || res.stage === ‘end‘) {
count -= 1;
console.log(res.id + ‘ ‘ + res.status + ‘ - ‘ + res.url);
if (count === 0) {
renderTimeout = setTimeout(doRender, resourceWait);
}
}
};

page.open(url, function (status) {
if (status !== "success") {
console.log(‘Unable to load url‘);
// phantom.exit();
} else {
forcedRenderTimeout = setTimeout(function () {
console.log(count);
doRender();
}, maxRenderWait);
}
});