python+selenium操作chrome浏览器抓取网页解决方案

Posted 2021-03-15 cknds

以下操作均是在ubuntu系统下运行

from selenium import webdriver
from scrapy.selector import Selector

#操作chrome浏览器抓取淘宝

driver = webdriver.Chrome()
driver.get(‘淘宝链接‘)
print(driver.page_source)
t_selector = Selector(text=driver.page_source)
tm_price = t_selector.xpath(‘//*[@id="J_StrPriceModBox"]/dd/span/text()‘).extract_first()
print(tm_price)
driver.quit()

#操作谷歌浏览器登录知户

browser = webdriver.Chrome()
browser.get("https://www.zhihu.com/signin?next=%2Fexplore")
browser.find_element_by_xpath(‘//div[@class="SignFlow-accountInput Input-wrapper"]/input[@name="username"]‘).send_keys("18262031725")
browser.find_element_by_xpath(‘//div[@class="Input-wrapper"]/input[@name="password"]‘).send_keys("....")
browser.find_element_by_xpath(‘//div[@class="Login-options"]/button[@type="submit"]‘).click()

#操作浏览器登录微薄，並且下拉

browser = webdriver.Chrome()
browser.get("https://weibo.com/")
browser.implicitly_wait(10)
browser.find_element_by_xpath(‘//input[@id="loginname"]‘).send_keys(‘18262031725‘)
browser.find_element_by_xpath(‘//div[@class="input_wrap"]/input[@type="password"]‘).send_keys(‘。。。。‘)
browser.find_element_by_xpath(‘//div[@class="info_list login_btn"]/a‘).click()
import time
time.sleep(5)
browser.execute_script("window.scrollTo(0,document.body.scrollHeight);var lenofPage=document.body.scrollHeight;return lenofPage")

#設置不加载图片

chrome_opt = webdriver.ChromeOptions()
prefs = {"profile.managed_default_content_settings.images":2} # 不加载图片
chrome_opt.add_experimental_option("prefs",prefs)
browser = webdriver.Chrome(chrome_options=chrome_opt)
browser.get("https://weibo.com/")

#phantomjs,无界面浏览器，多进程情况下phantomjs性能下降非常严重,不推荐用

driver = webdriver.PhantomJS()
driver.get(‘https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.1.1bb93598Z5K5Sz&id=45696640046&skuId=3361138965307&user_id=728443962&cat_id=2&is_b=1&rn=7fd2566c963598402926dd52a6776c8c‘)
print(driver.page_source)
t_selector = Selector(text=driver.page_source)
tm_price = t_selector.xpath(‘//*[@id="J_StrPriceModBox"]/dd/span/text()‘).extract_first()
print(tm_price)
driver.quit()

#通过PyVirtualDisplay可以实现无界面,需要安装xvfb

sudo apt-get install xvfb
pip install xvfbwrapper

from pyvirtualdisplay import Display
display = Display(visible=0,size=(800,600))
display.start()
browser = webdriver.Chrome()
browser.get(‘https://weibo.com/‘)
print(browser.page_source)

#推荐Headless Chrome指在headless模式下运行谷歌浏览器，实现无界面

opt = webdriver.ChromeOptions()
opt.set_headless()
opt.add_argument(‘--disable-gpu‘)
browser = webdriver.Chrome(options=opt)
browser.get("https://weibo.com/")
print(browser.page_source)
browser.close()