nodejs之爬虫

Posted 2022-08-10 rrrjc

//引入http
const http = require("http");
//引入https
const https = require("https");
//引入url
const url = require("url");
//引入path
const path = require("path");
//引入fs
const fs = require("fs");

//准备网址
const targetUrl = "http://www.nipic.com/photo/jianzhu/shinei/index.html?page=1";
//http或https的情况
const req = url.parse(targetUrl).protocol == "http:" ? http : https;
// console.log(req);
//get请求数据
req.get(targetUrl, (rs) => 
    //保存请求到的数据
    let str = "";
    //获取网页数据
    rs.on("data", (data) => 
        str += data;
    )
    //监听结束
    rs.on("end", () => 
        //定义正则
        let reg = /<img src="(.*?)" data-src="(.*?)"  alt="(.*?)" \/>/img;
        // 定义变量 用于保存每一次匹配的结果
        let result;
        // 定义一个存储图片地址的数组
        let imgarr = [];
        //遍历
        while ((result = reg.exec(str)) != null) 
            imgarr.push(result[2]);  // 把爬取到的图片地址放入一个数组
        
        // console.log(imgarr);
        
        //循环数组
        for(let i in imgarr)
            // console.log(imgarr[i]);
            // //再次请求拿到所有图片
            setTimeout(function()
                req.get(imgarr[i], (rs) => 
                    // 重命名文件
                    let rename = new Date().getTime()+path.extname(imgarr[i])
                    // console.log(rename);
                    
                    //创建写入流
                    let ws = fs.createWriteStream(`./download/$rename`);
                    rs.pipe(ws)
                    console.log("文件下载中");
                )
            ,100*i)
            
        
    )
)