nodejs 之 小爬虫

Posted 2020-09-27 北斗七七

一、简单的单页面

var http = require(\'http\') var url = \'http://www.imooc.com/learn/348\' 
http.get(url,function(res){ 
　　var html = \'\'
    //有data触发时
    res.on(\'data\',function(data){
        html += data
    })
    res.on(\'end\',function(){
        console.log(html)
    })
//出现异常时
}).on(\'error\',function(){
    console.log(\'获取出错\')
})

var server = http.createServer(function(req,res){
        res.writeHead(200,{\'Content-Type\':\'text/plain\'})
        res.end();
    })
server.listen(2017)

　　

运行结果

 

二、获取页面的课程列表

安装cheerio

cmd 执行命令 npm install  cheerio 然后就可以require cheerio

var http = require(\'http\')
var cheerio = require(\'cheerio\')
var url = \'http://www.imooc.com/learn/348\'

//过滤页面，获取到列表的名称与id等信息  与javascript方法一致
function filterChapters(html){
    var $ = cheerio.load(html)
    // console.log($)
    var chapters = $(\'.chapter\')

    var courseData = []
    chapters.each(function (item) {
        var chapter = $(this)
        var chapterTitle = chapter.find(\'strong\').text()
        var videos = chapter.find(\'.video\').children(\'li\')
        var chapterData = {
            chapterTitle:chapterTitle,
            video:[]
        }
        // console.log(videos + \'24\')
        videos.each(function(item){
            // var video = $(this).find(\'.studyvideo\')
            var videoTitle = $(this).text()
            var id = $(this).attr("data-media-id")

            chapterData.video.push({
                title:videoTitle,
                id:id
            })
        })
        courseData.push(chapterData)
    })
    return courseData
}
//将获取到的信息打印出来
function printCourseInfo(courseData) {
    courseData.forEach(function(item){
        var chapterTitle = item.chapterTitle
        item.video.forEach(function(video){
            console.log(\'【\' +video.id+\'】\' +video.title +\'\\n\')
        })
    })
}
http.get(url,function(res){
    var html = \'\'
    res.on(\'data\',function(data){
        html += data
    })
    res.on(\'end\',function(){
        //过滤页面信息
        // console.log(html)
        var courseData = filterChapters(html)
        printCourseInfo(courseData)
    })
}).on(\'error\',function(){
    console.log(\'获取出错\')
})

var server = http.createServer(function(req,res){
        res.writeHead(200,{\'Content-Type\':\'text/plain\'})
        res.end();
    })
server.listen(1337)

运行结果