基于Python爬取百度返回搜索结果（标题超链接简介）代码

Posted 2022-05-31 nufe_wwt

# -*- coding = utf-8 -*-
import requests  # 发送请求
from bs4 import BeautifulSoup  # 解析页面
import pandas as pd  # 存入数据
import os
from time import sleep
import random
import re

# 伪装浏览器请求头 主要就是 User-Agent 和 Cookie 记得根据自己的浏览器补全
headers = 
    'Accept': '',
    'Accept-Language': '',
    'Cache-Control': '',
    'Host': 'www.baidu.com',
    'User-Agent': "",
    'Cookie': ''



def baidu_search(v_keyword, v_result_file, v_max_page):
		"""
    :param v_keyword: 检索关键字
    :param v_result_file: 保存csv的文件名
    :param v_max_page: 爬取的页数
    :return: 
    """
    for page in range(v_max_page):
        print('开始爬取第页'.format(page+1))
        wait_seconds = random.uniform(1,2)  # 等待时长
        print('开始等待秒'.format(wait_seconds))
        sleep(wait_seconds) #等待
        url = 'https://www.baidu.com/s?wd=' + v_keyword + '&pn=' + str(page * 10)
        r = requests.get(url, headers=headers)
        r.encoding = 'utf-8'
        html = r.text
        # print(html)
        print('响应码是:'.format(r.status_code))
        soup = BeautifulSoup(html, 'html.parser')
        # print(soup.text)
        result_list = soup.find_all(class_='result c-container xpath-log new-pmd')
        # result_list = soup.find_all(class_='result c-container new-pmd')
        print('正在爬取:,共查询到个结果'.format(url, len(result_list)))
        title_list = []  # 标题
        real_url_list = []  # 真链
        desc_list = []  # 简介
        for result in result_list:
            title = result.find('a').text
            print('title is:',title)
            href = result.find('a')['href']
            # print(href)
            real_url = get_real_url(v_url=href)
            try:
                desc = result.find(class_='content-right_8Zs40').text
            except:
                desc = ""
            try:
                site = result.find(class_='c-showurl c-color-gray').text
            except:
                site = ""
            title_list.append(title)
            real_url_list.append(real_url)
            desc_list.append(desc)
        df = pd.DataFrame(
            
                'title': title_list,
                'url': real_url_list,
                'content': desc_list,
            
        )
        if os.path.exists(v_result_file):
            header = None
        else:
            header = ['title','url','content']  # csv文件标头
        df.to_csv(v_result_file, mode='a+', index=False, header=header, encoding='utf_8_sig')
        print('结果保存成功：'.format(v_result_file))
        sleep(random.uniform(4,5))


def get_real_url(v_url):
    r = requests.get(v_url, headers=headers, allow_redirects=False)  # 不允许重定向
    if r.status_code == 302:  # 如果返回302，就从响应头获取真实地址
        real_url = r.headers.get('Location')
    else:  # 否则从返回内容中用正则表达式提取出来真实地址
        real_url = re.findall("URL='(.*?)'", r.text)[0]
    print('real_url is:', real_url)
    return real_url


if __name__ == '__main__':
    search_keyword = '检索关键字'
    max_page = 50
    result_file = '.csv'.format(search_keyword)
    if os.path.exists(result_file):
        os.remove(result_file)
        print('结果文件()存在，已删除'.format(result_file))
    baidu_search(v_keyword=search_keyword, v_result_file=result_file, v_max_page=max_page)