node小爬虫

Posted 2020-10-17

const http = require(‘http‘);
const fs = require(‘fs‘);
const cheerio = require(‘cheerio‘);
const urlArr = [];
const imgArr = [];
for(var i=1 ; i<=33 ; i++){
    urlArr.push(‘http://www.imooc.com/course/list?page=‘+i);
}

function getImg() {
    urlArr.forEach(function(url) {
        http.get(url, function(res) {
            var html;
            res.on(‘data‘, function(data) {
                html += data;
            });
            res.on(‘end‘, function() {
                var $ = cheerio.load(html);
                $(‘.course-card‘).each(function(index, ele) {
                    var title = $(ele).find(‘.course-card-name‘).text();
                    var imgUrl = $(ele).find(‘.course-banner‘).attr(‘src‘);
                    var videoUrl = ‘http://www.imooc.com‘+$(ele).attr(‘href‘);
                    var text = title + videoUrl;
                    save_img(‘http:‘+imgUrl, title);
                    save_title(title, text);
                });
            });
        });
    });
}

function save_img(url, title) {
    http.get(url, function(res){
        var imgData = "";
        res.setEncoding("binary");
        res.on("data", function(chunk){
            imgData += chunk;
        });
        res.on("end", function(){
            fs.writeFile("node_download/img/"+title+".jpg", imgData, "binary", function(err){
                if(err){
                    console.log(err);
                }else {
                    console.log(title);
                }
            });
        });
    });
}

function save_title(title, text) {
    fs.writeFile("node_download/txt/"+title+".txt", text, ‘utf8‘,function(err){
        if(err){
            console.log(err);
        }else {
            console.log(title);
        }
    });
}

getImg();