selenium爬取壹共享平台

Posted 2020-11-30 wujf-myblog

#!usr/bin/env python
#-*- coding:utf-8 _*-
"""
@author:lenovo
@file: 壹共享.py
@time: 2019/11/08
"""
from selenium import webdriver
import time
from lxml import etree
import requests
import re
driver = webdriver.Chrome(r‘C:UserslenovoAppDataLocalGoogleChromeApplicationchromedriver.exe‘)
# 设置浏览器窗口的位置和大小
driver.set_window_position(200,0)
driver.set_window_size(1500,1000)
driver.get("http://www.igxpt.com/")
driver.save_screenshot(‘abort.png‘)

driver.find_element_by_xpath(‘//div[@class="input-wrap"]/input‘).clear()
driver.find_element_by_xpath(‘//div[@class="input-wrap"]/input‘).send_keys(‘U盘‘)
driver.find_element_by_xpath(‘//div[@class="input-wrap"]/button‘).click()

#得到第二页的链接

# driver.find_element_by_link_text("下一页")

current_url = driver.current_url

result  = requests.get(current_url)

content = result.text
html    = content.encode(‘utf-8‘).decode(‘utf-8‘)


‘‘‘开始爬取‘‘‘

ret        = etree.HTML(html)
total_text = ret.xpath(‘//div[@class="dataTables_paginate paging_simple_numbers"]/span[1]/text()‘)[0]
print(total_text)
number_str = re.findall(‘d+‘,total_text)
page       = int(number_str[0])

x = 1
for x in range(1,page+1):
	try:

		driver.find_element_by_link_text(‘下一页‘).click()
		print(‘当前页是‘+driver.current_url)
		#。。。。。。。do  something
	except Exception as e:
		print(e)


#退出窗口
driver.quit()

　随笔。。。。。。