php [robots.txt解析器] robots.txt php解析器#php

Posted 2021-05-04

<?php
/*
https://github.com/bopoda/robots-txt-parser
composer require bopoda/robots-txt-parser
*/
use RobotsTxtParser;

//parse robots.txt
$parser = new RobotsTxtParser(file_get_contents('http://example.com/robots.txt'));
var_dump($parser->getRules());

//validate URL, use the RobotsTxtValidator class
$validator = new RobotsTxtValidator($parser->getRules());

$url = '/';
$userAgent = 'MyAwesomeBot';

if ($validator->isUrlAllow($url, $userAgent)) {
    // Crawl the site URL and do nice stuff
}