javascript NodeJs ExpressJS中间件，允许特定于bot / crawler的路由。基于OS项目的Prerender.IO中间件用于节点。

Posted 2021-05-03

require("requirejs").define("middleware/onlyCrawlers", ["url"], function (url) {
    var crawlerUserAgents = [
//        'googlebot',
//        'yahoo',
//        'bingbot',
        'baiduspider',
        'facebookexternalhit',
        'twitterbot',
        'rogerbot',
        'linkedinbot',
        'embedly',
        'quora link preview',
        'showyoubot',
        'outbrain',
        'pinterest'
    ];

    return function (req, res, next) {
        var userAgent = req.headers['user-agent']
            , bufferAgent = req.headers['x-bufferbot']
            , isCrawler = false;

        if (!!userAgent && req.method == 'GET') {
            //if it contains _escaped_fragment_, show prerendered page
            if (url.parse(req.url, true).query.hasOwnProperty('_escaped_fragment_')) {
                isCrawler = true;
            }

            //if it is a bot...show prerendered page
            if (crawlerUserAgents.some(function (crawlerUserAgent) {
                return userAgent.toLowerCase().indexOf(crawlerUserAgent.toLowerCase()) !== -1;
            })) {
                isCrawler = true;
            }

            //if it is BufferBot...show prerendered page
            if (bufferAgent) {
                isCrawler = true;
            }

            if (isCrawler)
            {
                next();
            }
            else{
                next("route");
            }
        }
        else {
            next("route");
        }
    };
});