javascript 使用Headless-chorme-crawler（puppeteer）抓取并截取网站截图

Posted 2021-05-03

const HCCrawler = require('headless-chrome-crawler');
const CSVExporter = require('headless-chrome-crawler/exporter/csv');

const DOMAIN = 'example.com';
const TIMESTAMP = new Date().getTime();
const FILE = './tmp/result-' + TIMESTAMP + '.csv';
const URL = 'https://' + DOMAIN;

const exporter = new CSVExporter({
  file: FILE,
  fields: ['response.url', 'response.status', 'links.length'],
});

HCCrawler.launch({
  maxDepth: 2,
  exporter,
   preRequest: (options => {
    // if (!options.saveAs) return false; // Skip the request by returning false
    // console.log(options)
    let splitted_url = options.url.split('/')
    let page = splitted_url[splitted_url.length - 1];
    if(!page) {
      page = 'HOME-' + Math.floor(Math.random());
    }
    options.screenshot = { path: './tmp/IMG-' + splitted_url[2] + '-' + page + '-' + TIMESTAMP + '.png',
    fullPage: true};
    return true;
  }),
})
  .then(crawler => {
    crawler.queue({url: URL});
    crawler.onIdle()
      .then(() => crawler.close());
  });