哔哩哔哩舞蹈区每日前100名内容-Spider

Posted 2021-02-22 nuochengze

说明：无意滋生事端，仅学习分享，如有侵权，立即删除

用到的模块：json、lxml的etree、time.ctime、requests

源码如下：

import requests
import time
from lxml import etree
import json


class BiLiSpider():
    def __init__(self):
        self.url = ‘https://www.bilibili.com/ranking/all/129/0/3?spm_id_from=333.851.b_62696c695f7265706f72745f64616e6365.39‘
        self.headers = {
            ‘user-agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/81.0.4044.122 Safari/537.36‘,
        }

    def get_response(self):
        ret = requests.get(self.url,headers=self.headers)
        ret = ret.content.decode()
        return ret

    def make_content(self,ret1):
        all_item_list = list() # 创建一个列表用于接受字典，并用来排名
        for ret_ in ret1:
            item = dict() # 创建一个字典用于接收每名的标题、得分、链接地址
            all_item = dict() # 创建一个字典用于保存作者,并关联排名
            item["title"] = ret_.xpath(‘.//div[@class="info"]/a/text()‘)[0] if len(ret_.xpath(‘.//div[@class="info"]/a/text()‘)) else None
            item[‘hot_score‘] = ret_.xpath(‘.//div[@class="pts"]/div/text()‘)[0] if len(ret_.xpath(‘.//div[@class="pts"]/div/text()‘)) else None
            item["title_href"] = ret_.xpath(‘.//div[@class="info"]/a/@href‘)[0] if len(ret_.xpath(‘.//div[@class="info"]/a/@href‘)) else None
            author = ret_.xpath(‘.//div[@class="detail"]/a/span/text()‘)[0] if len(ret_.xpath(‘.//div[@class="detail"]/a/span/text()‘)) else None
            all_item["{}".format(author)] = item
            all_item_list.append(all_item)
        return all_item_list

    def save_file_response(self,ret):
        with open("spider_bilibil({}).html".format(time.ctime()),‘w‘,encoding="utf8") as f:
            f.write(ret)
        print("保存响应内容成功")

    def save_file(self,ret):
        with open("哔哩哔哩舞蹈区前100名内容({}).json".format(time.ctime()),‘w‘,encoding="utf-8") as f:
            f.write(json.dumps(ret,ensure_ascii=False,indent=2))
        print("保存哔哩哔哩舞蹈区前100名内容成功")

    def run(self):
        # 获取相应对象
        ret = self.get_response()
        # 保存response对象
        self.save_file_response(ret)
        # 实例化一个element对象
        html = etree.HTML(ret)
        # 对element对象分组
        ret1 = html.xpath("//ul[@class=‘rank-list‘]/li[@class=‘rank-item‘]")
        # 获取舞蹈区前100名的内容
        all_item = self.make_content(ret1)
        # 保存文件
        self.save_file(all_item)

        
if __name__ == ‘__main__‘:
    obj = BiLiSpider()
    obj.run()