Python爬虫采集青年大学习答案

Posted 2021-07-17 忆想不到的晖

引言

每周一期的青年大学习，采集答案到本地并获取最近一期的答案，下次就不用去百度寻找答案了。

直接 Python ys.py 获取

数据来源：http://www.quxiu.com/news/1692867.html

源代码

gevent 协程的使用，采集真的快。

"""
Author: Hui
Desc: { 青年大学习的答案爬取 }
"""
import os
import json
import gevent
from lxml import etree
from gevent import monkey

monkey.patch_all()
import request


class YoungStudy(object):
    save_path = 'YoungStudyAnswer.json'

    def __init__(self):
        self.domain = 'http://www.quxiu.com'
        self.url = 'http://www.quxiu.com/news/1692867.html'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/37.0.2049.0 Safari/537.36'
        }

        self.answer_list = self.load_answer_data()

    def load_answer_data(self):
        """
        先加载答案数据，好动态的添加新数据
        :return:
        """
        answers = list()
        if os.path.exists(self.save_path):
            with open(self.save_path, 'r', encoding='utf-8') as f:
                ret = f.read()
                if ret:
                    answers = json.loads(ret)
                return answers
        else:
            return answers

    def get_data(self, url):
        """
        获取url响应的数据
        """
        response = request.get(url, headers=self.headers)
        return response.content

    def parse_data(self, data):
        """
        解析首页，并获取所有答案的url
        :param data:
        :return: answer_urls
        """
        # print(data)
        data = data.decode()
        html = etree.HTML(data)
        hrefs = html.xpath("//td/a/@href")
        answer_urls = [self.domain + href for href in hrefs]
        return set(answer_urls)

    def parse_answer_data(self, data, url):
        """
        解析答案数据
        :param url: 当前解析的url，用于数据的封装
        :param data:
        :return: answer_dict
        """
        html = etree.HTML(data.decode())
        title = html.xpath('//title/text()')[0]
        create_date = html.xpath("//p[@class='source']/span[1]/text()")[0]
        # eg: 发布时间：2021-4-17 12:53:23
        create_date = create_date.split('：')[1]

        # 获取案例详情
        el_list = html.xpath("//div[@class='nawContent mb20']/p[position()>2]")
        answer = ''
        for el in el_list:
            # text()不能取到内嵌标签里的文本数据，改用 string(.)
            answer += el.xpath('string(.)') + '\\n'
        # print(answer)

        # 将答案信息封装在字典中
        answer_dict = dict()
        answer_dict['url'] = url
        answer_dict['title'] = title
        answer_dict['create_date'] = create_date
        answer_dict['answer'] = answer
        print(answer_dict)
        self.answer_list.append(answer_dict)
        return answer_dict

    def save_data(self):
        """
        把答案数据以 json 的格式保存
        :return:
        """
        with open(self.save_path, 'w', encoding='utf-8') as f:
            json.dump(self.answer_list, f, ensure_ascii=False, indent=2)

    def run(self):
        data = self.get_data(self.url)

        answer_urls = self.parse_data(data)

        print(f'answer_urls: {len(answer_urls)}')
        print(f'answer_list: {len(self.answer_list)}')

        if len(answer_urls) > len(self.answer_list):
            # 说明有新增数据
            task = []

            urls = [item['url'] for item in self.answer_list]
            # 集合求差值获取新的url(提取未解析过的url)
            new_urls = list(set(answer_urls) ^ set(urls))
            print(f'新增url {len(new_urls)} 条')
            for url in new_urls:
                # 给协程gevent添加多任务
                task.append(gevent.spawn(self.get_answer_task, url))

            gevent.joinall(task)
            self.save_data()
        else:
            print(len(answer_urls))
            print("无新增数据")

    def get_answer_task(self, url):
        """
        获取青年大学习答案
        :param url: 答案url
        :return:
        """
        answer_data = self.get_data(url)
        self.parse_answer_data(answer_data, url)

    def get_recent_answer(self):
        """
        获取最近一期答案
        :return:
        """
        print('-+-' * 10 + ' 最近一期答案 ' + '-+-' * 10 + '\\n')
        if self.answer_list:
            recent_answer = max(self.answer_list, key=lambda item: item['create_date'])
            print(recent_answer['title'])
            print(recent_answer['create_date'])
            print(recent_answer['url'])
            print(recent_answer['answer'])


def main():
    ys = YoungStudy()
    ys.run()
    ys.get_recent_answer()


if __name__ == '__main__':
    main()

公众号

新建文件夹X

大自然用数百亿年创造出我们现实世界，而程序员用几百年创造出一个完全不同的虚拟世界。我们用键盘敲出一砖一瓦，用大脑构建一切。人们把1000视为权威，我们反其道行之，捍卫1024的地位。我们不是键盘侠，我们只是平凡世界中不凡的缔造者 。