Promise队列应用实例(爬虫)

Posted 2021-11-25 IT飞牛

每隔一秒输出数组中的一个字符串，直到全部输出完成为止

var arr=["你好","hello","hi","good morning","nice to meet you"];
function run(str){
    //异步输出字符串
    return new Promise(function(resolve,reject){
        setTimeout(function(){
            console.log(str);
            resolve(str);
        },1000);
    });
}
//-----写法一-----
function queue(qs){
    let promise=Promise.resolve();
    qs.forEach((e,i,arr)=>{//这里的foreach可以理解为：将多个then操作直接串联起来，而不是每次替换上一个promise的值
        promise=promise.then(function(val){
            return run(e);
        });
    });
    return promise;
}

//-----写法二-----
function queue(qs){
    return qs.reduce((promise,e)=>{
        return promise.then(function(val){
            return run(e);
        });
    },Promise.resolve());
}

queue(arr).then(()=>{
    console.log("全部输出完成!");
});

爬虫代码

<script>
    //这是用Promise.then队列实现的一个网页爬虫功能；
    //data中返回的字段: title:string,url:string,children:[]
    //待优化：添加deep设置探测深度
    function fetch(url) {
        let delayTime = 1000;
        return new Promise((resolve, reject) => {
            setTimeout(() => {
                var xhr = new XMLHttpRequest();
                xhr.open("GET", url, true);
                xhr.responseType = "document";
                xhr.onreadystatechange = function () {
                    if (xhr.readyState === XMLHttpRequest.DONE && xhr.status == 200) {
                        var domTree = xhr.response;
                        var urls = Array.prototype.map.call(domTree.getElementsByTagName("a"), (e, i, arr) => {
                            return e.getAttribute("href");
                        });

                        if (urls.length > 0) {
                            var results = [];
                            queue(urls, results).then(() => {
                                resolve({
                                    title: domTree.title,
                                    status: xhr.status,
                                    url: url,
                                    children: results,
                                });
                            });
                        } else {
                            resolve({
                                title: domTree.title,
                                status: xhr.status,
                                url: url,
                                children: [],
                            });
                        }
                    } else if (xhr.readyState === XMLHttpRequest.DONE && xhr.status != 200) {
                        resolve({
                            title: "错误页面",
                            status: xhr.status,
                            url: url,
                            children: [],
                        });
                    }
                }
                xhr.send();
            }, delayTime);
        });
    }

    function queue(urls, D) {
        return urls.reduce((promise, url) => {
            return promise.then(() => {
                return new Promise((r, j) => {
                    fetch(url).then((data) => {
                        D.push(data);
                        r();
                    });
                });
            });
        }, Promise.resolve());
    }

    fetch(location.href).then((data) => {
        console.log(data);
    });
</script>

pachong.html中有p1、p2、p3三个页面链接
p1中有p11、p12、p13三个链接