爬百度新闻

Posted 2021-03-04 li128

from selenium import webdriver

from bs4 import BeautifulSoup

from selenium.webdriver.common.by import By

from selenium.webdriver.common.keys import Keys

import time

 infor_name= [ ] #标题

infor_url=[ ] #网址

browser = webdriver.Chrome(‘C:Program Files (x86)GoogleChromeApplicationchromedriver.exe‘)

 find_name=[‘激光雷达‘,‘lidar‘]

num=len(find_name)

def find_infor(htmlin):

xpath_urls = ‘//h3[@class="c-title"]/a‘

texts = browser.find_elements_by_xpath(xpath_urls)

 for text_xx in texts:

infor_url.append(text_xx.get_attribute("href"))

infor_name.append(text_xx.text)

 for x in range(num):

browser.get("百度新闻搜索--全球最大的中文新闻平台")

# print(browser.title)

assert "百度新闻搜索——全球最大的中文新闻平台" in browser.title

browser.find_element_by_id("ww").send_keys(find_name[x])

browser.find_element_by_class_name("btn").click()

 time.sleep(3) # 给网页读取的时间，否则是空白网页信息

find_infor(browser)

 browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")

browser.find_element_by_xpath("//a[@class=‘n‘]").click()

time.sleep(3)

find_infor(browser)

 browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")

browser.find_element_by_xpath("//a[@class=‘n‘]").click()

time.sleep(3)

find_infor(browser)

 browser.quit()

 print(infor_name)

print(infor_url)