爬虫：获取动态加载数据（selenium）（某站）

Posted 2021-05-19 tree1000

　　如果网站数据是动态加载，需要不停往下拉进度条才能显示数据，用selenium模拟浏览器下拉进度条可以实现动态数据的抓取。

　　本文希望找到某乎某话题下讨论较多的问题，以此再寻找每一问题涉及的话题关键词（侵删）。

　　下面代码采用driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")模拟浏览器下拉进度条200次，获取了女性话题下近900多条回答，去重（同一话题下有重复问题）后得到600多个问题

from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions
import time
import random
import re
from pymongo import MongoClient

client = MongoClient(\'localhost\')
db = client[\'test_db\']


def get_links(url, word):
    option = ChromeOptions()
    option.add_experimental_option(\'excludeSwitches\', [\'enable-automation\'])
    option.add_argument(\'user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/79.0.3945.130 Safari/537.36"\')
    driver = Chrome(options=option)
    time.sleep(10)
    driver.get(\'https://www.zhihu.com\')
    time.sleep(10)
    driver.delete_all_cookies()  # 清除刚才的cookie
    time.sleep(2)
    cookie = {}  # 替换为自己的cookie
    driver.add_cookie(cookie)
    driver.get(url)
    time.sleep(random.uniform(10, 11))

    for i in range(0, 200):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(random.uniform(3, 4))

    links = driver.find_elements_by_css_selector(\'h2.ContentItem-title > div > a\')
    print(len(links))

    # 去重
    regex = re.compile(r\'https://www.zhihu.com/question/\\d+\')  # 匹配问题而不是答案链接
    links_set = set()
    for link in links:
        try:
            links_set.add(regex.search(link.get_attribute("href")).group())
        except AttributeError:
            pass
    print(len(links_set))

    with open(r\'知乎镜像链接\' + \'/\' + word + \'-\'.join(str(i) for i in list(time.localtime())[:5]) + \'.txt\', \'a\') as f:
        for item in links_set:
            f.write(item + \'\\n\')
            db[word + \'_\' + \'links\'].insert_one({"link": item})


if __name__ == \'__main__\':
    input_word = input(\'输入话题：\')
    input_link = input(\'输入话题对应链接url：\')
    get_links(input_link, input_word)

截图如下：