全国图书馆参考咨询联盟模拟登陆及爬取可爬取的图片
Posted chka
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了全国图书馆参考咨询联盟模拟登陆及爬取可爬取的图片相关的知识，希望对你有一定的参考价值。
一、编程思路
1.模拟登陆 采用selenium PhantomJS 采用Chrome Firefox 这些，我的电脑无法截取验证码位置，读者可以自行尝试 验证码识别可采用tesserocr 我采用手动输入
2、查询，获取搜索框，用户输入关键字并查询
3、页面信息，F12查看即可 ，若采用find_element_by_xpath()查询需注意element 返回是第一个节点信息 elements返回是一个列表
4、书本具体信息、F12查看，后面操作很简单
5、文献传递页面、这个地方是最难的，右键查看文献传递这个按钮，点击其中href是无法进入的，这个只是一种绑定关系，需要仔细观察进入文献传递页面前后的network中第一个文本中的信息，里面存在很多url，
只有refer 点击可以进入，分析refer url里面的元素，在进入前的那个页面的url可找到，后面采用切片即可
6、下载书名页...............，此处我采用的是观察图片的链接直接的关系，从而获取，这个地方需要注意的是，图片需要不断的滑动滑动条才能加载图片，否则无法下载
7、保存图片 ，注意 ‘w‘ 和‘wb’使用即可
8、最后需要注意爬取频率，否则很容易被发现。
from selenium import webdriver import time #import tesserocr import pytesseract from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import Select from PIL import Image import requests from selenium.webdriver.common.desired_capabilities import DesiredCapabilities # from pyquery import PyQuery as pq from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from lxml import etree from urllib.parse import quote import os # cookie_bro = browser.get_cookies() # # cookie1=cookie_bro[1][‘value‘] # print(‘获取cookie成功‘) def login(): #输入用户名 input_user = browser.find_element_by_id("userName") #查找输入框 input_user.send_keys("ckho") time.sleep(2) #输入密码 input_pwd = browser.find_element_by_id("passWord") input_pwd.send_keys("chen13560034370") time.sleep(2) #选择 input_gid = Select(browser.find_element_by_id("gid")) #实例化input_gid input_gid.select_by_value("7320") #选择class=7320的选择 browser.save_screenshot("screen.png") # 定位验证码图片的位置，并截取该位置的图片 code_element = browser.find_element_by_id("vimg") print(code_element.location) # {‘x‘: 548, ‘y‘: 523} left = code_element.location[‘x‘] top = code_element.location[‘y‘] right = code_element.size[‘width‘] + left height = code_element.size[‘height‘] + top im = Image.open("screen.png") img = im.crop((left, top, right, height)) img.save("screen4.png") #验证码识别 try: input_verify = wait.until( EC.element_to_be_clickable((By.ID,"verifyCode")) )#browser.find_element_by_id("verifyCode") result = input("请输入验证码") input_verify.send_keys(result) #单击登录 enter = wait.until( EC.element_to_be_clickable((By.ID,"submit")) )#browser.find_element_by_id("submit").click() enter.click() print("登录成功") browser.save_screenshot("screen6.png") return browser.current_url except BaseException: print(" Enter Error") #查询书籍信息并且用户可选择页数 def index_page(url): book_name = input("请输入查找的书名") input_bookname = browser.find_element_by_id("sw").send_keys(book_name) enter = browser.find_element_by_xpath(‘//*[@id="f2"]/div[2]/input[1]‘).click() print("当前页数为第一页") all_book_information() page = input("请输入想看页数：") print("...正在爬取第"+str(page)+"页") current_url = browser.current_url #输入书名后的页面的链接 try: if int(page)>1: browser.get(current_url) print(current_url) #查找输入框 input_page = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, "#jpage")) ) #查找登录建 sumbit = wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, ‘#pageinfo > input[type=button]:nth-child(13)‘)) ) input_page.clear() input_page.send_keys(str(page)) sumbit.click() all_book_information() return browser.current_url #换页后的链接 except TimeoutError: index_page() def all_book_information(): #返回当前页面的书本信息 addres =‘//table[@class="book1"]‘ addres_list = browser.find_elements_by_xpath(addres) book_list = [] for book in addres_list: book_list.append(book.text) for i in enumerate(book_list, start=1): print(i) #获取每本书具体链接并且返回每本书具体信息 def get_detail_book(url): number = input("请输入你想要了解书的编号：") browser.get(url) addres = ‘//table[{}][@class="book1"]//a[@class="px14"]‘.format(number) book_url = browser.find_element_by_xpath(addres).get_attribute("href") browser.get(book_url) detail_book_information = browser.find_elements_by_xpath(‘//div[@class="tubox"]//dd‘) for book in detail_book_information: print(book.text) return browser.current_url #进入图书馆文献传递页面 def sent_book_emial(url): bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") all_page_name(bqy_url) answer = input("是否需要下载此书 是请输入是1 否2 看其他书按3 ,下载书名页4 下载前言页5 下载版权页6 下载目录页7 下载正文页8") if int(answer)==1: base_url = ‘http://book.ucdrs.superlib.net/gofirstdrs.jsp?‘ browser.get(url) sent_href = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[2]//a‘).get_attribute("href") #页面上文献传递对应href值 #拆分href，拼接可进入图书参考咨询页面的url 通过进入文献传递，观察network中找到可进入图书参考咨询页面url，直接点击href是不能访问 list1 = sent_href.split("?", 1) list2 = list1[1].split("‘", 1) tscx_url = base_url+list2[0] browser.get(tscx_url) browser.save_screenshot("screen5.png") book_download() elif int(answer)==2: print(" ") print("本次查询结束，欢迎下次使用!") elif int(answer) == 4: browser.get(url) bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") base_url = bqy_download(bqy_url) smy_img(base_url) elif int(answer)==5: browser.get(url) bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") base_url = bqy_download(bqy_url) qyy_img(base_url) elif int(answer)==6: browser.get(url) bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") base_url = bqy_download(bqy_url) bqy_url(base_url) elif int(answer)==7: browser.get(url) bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") base_url = bqy_download(bqy_url) mly_img(base_url) elif int(answer)==8: browser.get(url) bqy_url = browser.find_element_by_xpath(‘//*[@id="libinfo"]/div[1]/div/a[1]‘).get_attribute("href") base_url = bqy_download(bqy_url) zwy_img(base_url) else: url = "http://www.ucdrs.superlib.net/" browser.get(url) all_book_url_page = index_page(url) detail_book_url = get_detail_book(all_book_url_page) sent_book_emial(detail_book_url) def all_page_name(url): browser.get(url) all_page_name = browser.find_element_by_id("pagejump") t1 = all_page_name.text.replace(" ", "") #删除空格 print("该书可看部分仅有:"+t1+"请按照此选择下载，否则可能导致下载错误") #图书下载 def book_download(): all_page = browser.find_element_by_xpath(‘//*[@id="content"]/form/ul/li[3]/p[1]‘).text print(all_page) print("每本图书咨询每次不超过50页") input1 = input("请输入想看的书初始页") input2 = input("请输入想看的书的末页") input_start = browser.find_element_by_id("frompage").send_keys(input1) input_led = browser.find_element_by_id("endpage").send_keys(input2) email = input("请输入你的邮箱账号") input_email = browser.find_element_by_id("email").send_keys(email) verifycode1 = input("请输入验证码") input_verifycode1 = browser.find_element_by_id("verifycode").send_keys(verifycode1) input_enter = browser.find_element_by_xpath(‘//li[@class="sumbit"]‘).click() #返回图片的url共同部分 def bqy_download(url): browser.get(url) print(url) time.sleep(4) #注意 需要留个页面加载时间，模仿人阅读时候网页加载速度 否则加载不出来想要的图片链接 browser.save_screenshot("screen8.png") first_img_url = browser.find_element_by_xpath(‘//*[@id="reader"]/div/div[1]/input‘).get_attribute("src") print(first_img_url) base_url = first_img_url[0:-13] print(base_url) return base_url #下载书名页 def smy_img(base_url): i=1 print("仅下载1页") while i<2: img_url = base_url + ‘bok00{}‘.format(i) + ‘?zoom=0&f=0‘ i += 1 response = requests.get(img_url) print(img_url) with open("D:/pycharm/实战案例/前言页/" + str(i-1) + ‘.png‘, "wb") as f: f.write(response.content) print("success download") time.sleep(2) #下载版权页 def bqy_img(base_url): i=1 print("仅下载1页") while i<2: img_url = base_url + ‘leg00{}‘.format(i) + ‘?zoom=0&f=0‘ i += 1 response = requests.get(img_url) print(img_url) with open("D:/pycharm/实战案例/版权页/" + str(i-1) + ‘.png‘, "wb") as f: f.write(response.content) print("success download") #下载前言页 def qyy_img(base_url): i=1 print("仅下载5页") while i<6: img_url = base_url + ‘fow00{}‘.format(i) + ‘?zoom=0&f=0‘ i += 1 response = requests.get(img_url) print(img_url) with open("D:/pycharm/实战案例/前言页/" + str(i-1) + ‘.png‘, "wb") as f: f.write(response.content) print("success download") # try: # response.headers["Accept-Encoding"] # except: # break time.sleep(2) #下载目录页 def mly_img(base_url): i=1 print("仅下载3页") while i<4: img_url = base_url + ‘!0000{}‘.format(i) + ‘?zoom=0&f=0‘ i += 1 response = requests.get(img_url) print(img_url) with open("D:/pycharm/实战案例/目录页/" + str(i-1) + ‘.png‘, "wb") as f: f.write(response.content) print("success download") time.sleep(2) #下载正文页 def zwy_img(base_url): i=1 print("仅下载15页") while i<12: if i<16: img_url = base_url + ‘00000{}‘.format(i) + ‘?zoom=0&f=0‘ else: img_url = base_url + ‘0000{}‘.format(i) + ‘?zoom=0&f=0‘ i += 1 response = requests.get(img_url) print(img_url) with open("D:/pycharm/实战案例/正文页/" + str(i-1) + ‘.png‘, "wb") as f: f.write(response.content) print("success download") time.sleep(2) if __name__ == ‘__main__‘: url = "http://www.ucdrs.superlib.net/login/login.action" # headers = { # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", # "Accept-Encoding": "gzip, deflate", # "Accept-Language": "zh-CN,zh;q=0.9", # "Cache-Control": "max-age=0", # "Connection": "keep-alive", # "Cookie": "JSESSIONID=E9B8FFC8B023F0FC12A07A3ECDE91581.jp26; __dxca=d04d4dbb-57fb-4114-b080-190507ee4cbf; route=5ead36b501ee59635125fd6ef4221d0e; UM_distinctid=170b290e53493e-0372017c841b37-4313f6a-144000-170b290e5364a3; CNZZDATA2088844=cnzz_eid%3D382770322-1583543803-%26ntime%3D1583551301", # "Host": "img.duxiu.com", # "Upgrade-Insecure-Requests":"1", # "User-Agent":‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36‘ # } # # cookies = { # "Cookie": "JSESSIONID=E9B8FFC8B023F0FC12A07A3ECDE91581.jp26; __dxca=d04d4dbb-57fb-4114-b080-190507ee4cbf; route=5ead36b501ee59635125fd6ef4221d0e; UM_distinctid=170b290e53493e-0372017c841b37-4313f6a-144000-170b290e5364a3; CNZZDATA2088844=cnzz_eid%3D382770322-1583543803-%26ntime%3D1583551301", # } browser = webdriver.PhantomJS() browser.get(url) wait = WebDriverWait(browser, 8) print("欢迎使用图书查询小程序") login() # 登录 需要手动验证码 all_book_url_page = index_page(url) # 查看第几页及返回当前页的书本信息 detail_book_url = get_detail_book(all_book_url_page) #返回每本书的具体的链接 sent_book_emial(detail_book_url)
若有错误，请留言告诉我，谢谢！
以上是关于全国图书馆参考咨询联盟模拟登陆及爬取可爬取的图片的主要内容，如果未能解决你的问题，请参考以下文章