php中curl爬虫 怎么样通过网页获取所有链接

Posted 2023-03-20

本文承接上面两篇，本篇中的示例要调用到前两篇中的函数，做一个简单的URL采集。一般php采集网络数据会用file_get_contents、file和cURL。不过据说cURL会比file_get_contents、file更快更专业，更适合采集。今天就试试用cURL来获取网页上的所有链接。示例如下：

<?php
/*
* 使用curl 采集hao123.com下的所有链接。
*/
include_once('function.php');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'http://www.hao123.com/');
// 只需返回HTTP header
curl_setopt($ch, CURLOPT_HEADER, 1);
// 页面内容我们并不需要
// curl_setopt($ch, CURLOPT_NOBODY, 1);
// 返回结果，而不是输出它
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$html = curl_exec($ch);
$info = curl_getinfo($ch);
if ($html === false)
echo "cURL Error: " . curl_error($ch);

curl_close($ch);
$linkarr = _striplinks($html);
// 主机部分，补全用
$host = 'http://www.hao123.com/';
if (is_array($linkarr))
foreach ($linkarr as $k => $v)
$linkresult[$k] = _expandlinks($v, $host);


printf("<p>此页面的所有链接为：</p><pre>%s</pre>n", var_export($linkresult , true));
?>

function.php内容如下（即为上两篇中两个函数的合集）：

<?php
function _striplinks($document)
preg_match_all("'<s*as.*?hrefs*=s*(["'])?(?(1) (.*?)\1 | ([^s>]+))'isx", $document, $links);
// catenate the non-empty matches from the conditional subpattern
while (list($key, $val) = each($links[2]))
if (!empty($val))
$match[] = $val;
while (list($key, $val) = each($links[3]))
if (!empty($val))
$match[] = $val;

// return the links
return $match;

/*===================================================================*
Function: _expandlinks
Purpose: expand each link into a fully qualified URL
Input: $links the links to qualify
$URI the full URI to get the base from
Output: $expandedLinks the expanded links
*===================================================================*/
function _expandlinks($links,$URI)

$URI_PARTS = parse_url($URI);
$host = $URI_PARTS["host"];
preg_match("/^[^?]+/",$URI,$match);
$match = preg_replace("|/[^/.]+.[^/.]+$|","",$match[0]);
$match = preg_replace("|/$|","",$match);
$match_part = parse_url($match);
$match_root =
$match_part["scheme"]."://".$match_part["host"];
$search = array( "|^http://".preg_quote($host)."|i",
"|^(/)|i",
"|^(?!http://)(?!mailto:)|i",
"|/./|",
"|/[^/]+/../|"
);
$replace = array( "",
$match_root."/",
$match."/",
"/",
"/"
);
$expandedLinks = preg_replace($search,$replace,$links);
return $expandedLinks;

?> 参考技术A 你注意看下被过滤掉的关键词是否是中文，注意下编码格式。 参考技术B 获取到数据后 然后正则吧 最方便了 参考技术C <?php
/*
* 使用curl 采集hao123.com下的所有链接。
*/
include_once('function.php');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, '');
// 只需返回HTTP header
curl_setopt($ch, CURLOPT_HEADER, 1);
// 页面内容我们并不需要
// curl_setopt($ch, CURLOPT_NOBODY, 1);
// 返回结果，而不是输出它
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$html = curl_exec($ch);
$info = curl_getinfo($ch);
if ($html === false)
echo "cURL Error: " . curl_error($ch);

curl_close($ch);
$linkarr = _striplinks($html);
// 主机部分，补全用
$host = '';
if (is_array($linkarr))
foreach ($linkarr as $k => $v)
$linkresult[$k] = _expandlinks($v, $host);


printf("<p>此页面的所有链接为：</p><pre>%s</pre>n", var_export($linkresult , true));
?>

function.php内容如下（即为上两篇中两个函数的合集）：

<?php
function _striplinks($document)
preg_match_all("'<s*as.*?hrefs*=s*(["'])?(?(1) (.*?)\1 | ([^s>]+))'isx", $document, $links);
// catenate the non-empty matches from the conditional subpattern
while (list($key, $val) = each($links[2]))
if (!empty($val))
$match[] = $val;
while (list($key, $val) = each($links[3]))
if (!empty($val))
$match[] = $val;

// return the links
return $match;

/*===================================================================*
Function: _expandlinks
Purpose: expand each link into a fully qualified URL
Input: $links the links to qualify
$URI the full URI to get the base from
Output: $expandedLinks the expanded links
*===================================================================*/
function _expandlinks($links,$URI)

$URI_PARTS = parse_url($URI);
$host = $URI_PARTS["host"];
preg_match("/^[^?]+/",$URI,$match);
$match = preg_replace("|/[^/.]+.[^/.]+$|","",$match[0]);
$match = preg_replace("|/$|","",$match);
$match_part = parse_url($match);
$match_root =
$match_part["scheme"]."://".$match_part["host"];
$search = array( "|^http://".preg_quote($host)."|i",
"|^(/)|i",
"|^(?!http://)(?!mailto:)|i",
"|/./|",
"|/[^/]+/../|"
);
$replace = array( "",
$match_root."/",
$match."/",
"/",
"/"
);
$expandedLinks = preg_replace($search,$replace,$links);
return $expandedLinks;

?> 参考技术D 可以考虑用phpQuery这个类插件！具体，可以百度查下