Promise队列应用实例(爬虫)

Posted 2022-01-29 IT飞牛

每隔一秒输出数组中的一个字符串，直到全部输出完成为止

var arr=["你好","hello","hi","good morning","nice to meet you"];
function run(str)
    //异步输出字符串
    return new Promise(function(resolve,reject)
        setTimeout(function()
            console.log(str);
            resolve(str);
        ,1000);
    );

//-----写法一-----
function queue(qs)
    let promise=Promise.resolve();
    qs.forEach((e,i,arr)=>//这里的foreach可以理解为：将多个then操作直接串联起来，而不是每次替换上一个promise的值
        promise=promise.then(function(val)
            return run(e);
        );
    );
    return promise;


//-----写法二-----
function queue(qs)
    return qs.reduce((promise,e)=>
        return promise.then(function(val)
            return run(e);
        );
    ,Promise.resolve());


queue(arr).then(()=>
    console.log("全部输出完成!");
);

爬虫代码

<script>
    //这是用Promise.then队列实现的一个网页爬虫功能；
    //data中返回的字段: title:string,url:string,children:[]
    //待优化：添加deep设置探测深度
    function fetch(url) 
        let delayTime = 1000;
        return new Promise((resolve, reject) => 
            setTimeout(() => 
                var xhr = new XMLHttpRequest();
                xhr.open("GET", url, true);
                xhr.responseType = "document";
                xhr.onreadystatechange = function () 
                    if (xhr.readyState === XMLHttpRequest.DONE && xhr.status == 200) 
                        var domTree = xhr.response;
                        var urls = Array.prototype.map.call(domTree.getElementsByTagName("a"), (e, i, arr) => 
                            return e.getAttribute("href");
                        );

                        if (urls.length > 0) 
                            var results = [];
                            queue(urls, results).then(() => 
                                resolve(
                                    title: domTree.title,
                                    status: xhr.status,
                                    url: url,
                                    children: results,
                                );
                            );
                         else 
                            resolve(
                                title: domTree.title,
                                status: xhr.status,
                                url: url,
                                children: [],
                            );
                        
                     else if (xhr.readyState === XMLHttpRequest.DONE && xhr.status != 200) 
                        resolve(
                            title: "错误页面",
                            status: xhr.status,
                            url: url,
                            children: [],
                        );
                    
                
                xhr.send();
            , delayTime);
        );
    

    function queue(urls, D) 
        return urls.reduce((promise, url) => 
            return promise.then(() => 
                return new Promise((r, j) => 
                    fetch(url).then((data) => 
                        D.push(data);
                        r();
                    );
                );
            );
        , Promise.resolve());
    

    fetch(location.href).then((data) => 
        console.log(data);
    );
</script>

pachong.html中有p1、p2、p3三个页面链接
p1中有p11、p12、p13三个链接