javascript 网站刮刀

Posted 2021-05-04

/*

https://github.com/website-scraper/node-website-scraper
https://www.npmjs.com/package/website-scraper-2
https://scraper.nepochataya.pp.ua/

npm install website-scraper
*/

var scraper = require('website-scraper');
scraper({
  urls: [
    'http://rsworldpi.com/'
  ],
  recursive: true,
  maxRecursiveDepth: 1,
  filenameGenerator: 'bySiteStructure',
  directory: 'rsworldpi',
  subdirectories: [
    {directory: 'img', extensions: ['.jpg', '.png', '.svg']},
    {directory: 'js', extensions: ['.js']},
    {directory: 'css', extensions: ['.css']}
  ],
  sources: [
    {selector: 'img', attr: 'src'},
    {selector: 'link[rel="stylesheet"]', attr: 'href'},
    {selector: 'script', attr: 'src'}
  ],
  request: {
    headers: {
      'User-Agent': 'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 4 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19'
    }
  }
}).then(function (result) {
  console.log('hello');
}).catch(function(err){
  console.log('fail' + err);
});