javascript JS爬虫为所有'nodeschool'研讨会获取描述和NPM包。

Posted 2021-05-03

/*
  "dependencies": {
    "cheerio": "^0.22.0",
    "request": "^2.79.0"
  }
*/

const request = require('request');
const cheerio = require('cheerio');

const SITE_URL = 'https://nodeschool.io';
const SEPARATOR = '-'.repeat(100);

function Crawler(url) {
  this.url = url;

  this.start = function() {
    request(this.url, function(error, response, body) {
      if(error) {
        console.log('Error:', error.message);
        return;
      }

      const $ = cheerio.load(body);
      const elements = $('.workshopper');

      for(var i = 0; i < elements.length; i++) {
        $ele = $(elements[i]);

        if($ele.attr('id')){
          var npmCodes = $ele.find('code');

          for(var j = 0; j < npmCodes.length; j++) {
            $npmCode = $(npmCodes[j]);

            if($npmCode.parent().hasClass('workshopper')) {
              console.log(SEPARATOR);
              console.log($ele.find('p').text());
              console.log($npmCode.text());
            }
          }
        }
      }
    });
  }
}

crawler = new Crawler(SITE_URL);
crawler.start();