爬虫入门
Posted qq849784670
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了爬虫入门相关的知识，希望对你有一定的参考价值。
1.处理验证码
云打码平台 注册：普通用户和开发用户 登录： 登录普通用户（查看余额） 登录开发者用户： 创建一个软件：我的软件->创建软件 下载实例代码：开发者中心->下载最新的DLL->pythonHttp实例代码下载
import http.client, mimetypes, urllib, json, time, requests ###################################################################### class YDMHttp: apiurl = ‘http://api.yundama.com/api.php‘ username = ‘‘ password = ‘‘ appid = ‘‘ appkey = ‘‘ def __init__(self, username, password, appid, appkey): self.username = username self.password = password self.appid = str(appid) self.appkey = appkey def request(self, fields, files=[]): response = self.post_url(self.apiurl, fields, files) response = json.loads(response) return response def balance(self): data = {‘method‘: ‘balance‘, ‘username‘: self.username, ‘password‘: self.password, ‘appid‘: self.appid, ‘appkey‘: self.appkey} response = self.request(data) if (response): if (response[‘ret‘] and response[‘ret‘] < 0): return response[‘ret‘] else: return response[‘balance‘] else: return -9001 def login(self): data = {‘method‘: ‘login‘, ‘username‘: self.username, ‘password‘: self.password, ‘appid‘: self.appid, ‘appkey‘: self.appkey} response = self.request(data) if (response): if (response[‘ret‘] and response[‘ret‘] < 0): return response[‘ret‘] else: return response[‘uid‘] else: return -9001 def upload(self, filename, codetype, timeout): data = {‘method‘: ‘upload‘, ‘username‘: self.username, ‘password‘: self.password, ‘appid‘: self.appid, ‘appkey‘: self.appkey, ‘codetype‘: str(codetype), ‘timeout‘: str(timeout)} file = {‘file‘: filename} response = self.request(data, file) if (response): if (response[‘ret‘] and response[‘ret‘] < 0): return response[‘ret‘] else: return response[‘cid‘] else: return -9001 def result(self, cid): data = {‘method‘: ‘result‘, ‘username‘: self.username, ‘password‘: self.password, ‘appid‘: self.appid, ‘appkey‘: self.appkey, ‘cid‘: str(cid)} response = self.request(data) return response and response[‘text‘] or ‘‘ def decode(self, filename, codetype, timeout): cid = self.upload(filename, codetype, timeout) if (cid > 0): for i in range(0, timeout): result = self.result(cid) if (result != ‘‘): return cid, result else: time.sleep(1) return -3003, ‘‘ else: return cid, ‘‘ def report(self, cid): data = {‘method‘: ‘report‘, ‘username‘: self.username, ‘password‘: self.password, ‘appid‘: self.appid, ‘appkey‘: self.appkey, ‘cid‘: str(cid), ‘flag‘: ‘0‘} response = self.request(data) if (response): return response[‘ret‘] else: return -9001 def post_url(self, url, fields, files=[]): for key in files: files[key] = open(files[key], ‘rb‘); res = requests.post(url, files=files, data=fields) return res.text ###################################################################### # 用户名 （普通用户） username = ‘lianda0707‘ # 密码 password = ‘ld18513185949.07‘ # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！ appid = 6998 # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！ appkey = ‘cf4dda0913c13bb707a251a847bce5a1‘ # 图片文件 filename = ‘getimage.jpg‘ # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html codetype = 1004 # 超时时间，秒 timeout = 10 # 检查 if (username == ‘username‘): print(‘请设置好相关参数再测试‘) else: # 初始化 yundama = YDMHttp(username, password, appid, appkey) # 登陆云打码 uid = yundama.login(); print(‘uid: %s‘ % uid) # 查询余额 balance = yundama.balance(); print(‘balance: %s‘ % balance) # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果 cid, result = yundama.decode(filename, codetype, timeout); print(‘cid: %s, result: %s‘ % (cid, result)) ######################################################################
def get_code(username, pwd, filepath, codetype): # 用户名 （普通用户） username = username # 密码 password = pwd # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！ appid = 6998 # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！ appkey = ‘cf4dda0913c13bb707a251a847bce5a1‘ # 图片文件 filename = filepath # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html codetype = codetype # 超时时间，秒 timeout = 20 result = None # 检查 if (username == ‘username‘): print(‘请设置好相关参数再测试‘) else: # 初始化 yundama = YDMHttp(username, password, appid, appkey) # 登陆云打码 uid = yundama.login(); print(‘uid: %s‘ % uid) # 查询余额 balance = yundama.balance(); print(‘balance: %s‘ % balance) # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果 cid, result = yundama.decode(filename, codetype, timeout); print(‘cid: %s, result: %s‘ % (cid, result)) return result # 人人网的模拟登录 import requests import urllib from lxml import etree # 获取session对象 session = requests.Session() # 将验证码图片下载 url = ‘http://www.renren.com/‘ headers = { # ‘Connection‘: ‘close‘, ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36‘ } page_text = requests.get(url=url, headers=headers).text tree = etree.HTML(page_text) code_img_url = tree.xpath(‘//*[@id="verifyPic_login"]/@src‘)[0] urllib.request.urlretrieve(url=code_img_url, filename=‘code.jpg‘) # 识别验证码图片中的数据值 code_data = get_code(‘lianda0707‘, ‘ld18513185949.07‘, ‘./code.jpg‘, 2004) # 模拟登录 login_url = ‘http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019141628905 ‘ data = { "email":"15764551436", "icode":code_data, "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":"96afa5f31bb9fb28ecc4c5eeda29a68b5aac63e38c3d51360ddba0b0f63e3238", "rkey":"5f7a1d96ff48551fe2910f08a1a2ddcd", "f":"", } #该次请求产生的cookie会被自动存储到session对象中 ll = session.post(url=login_url,data=data,headers=headers).text print(ll) url = ‘http://www.renren.com/969892564/profile‘ page_text = session.get(url=url,headers=headers).text with open(‘renren.html‘,‘w‘,encoding=‘utf-8‘) as fp: fp.write(page_text)
# 模拟登录古诗文忘 s = requests.Session() headers = { ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36‘ } login_url = ‘https://so.gushiwen.org/user/login.aspx?from=‘ page_text = s.get(url=login_url, headers=headers).text tree = etree.HTML(page_text) img_src = ‘https://so.gushiwen.org‘ + tree.xpath(‘//*[@id="imgCode"]/@src‘)[0] img_data = s.get(url=img_src, headers=headers).content with open(‘./img.jpg‘,‘wb‘) as fp: fp.write(img_data) img_text = get_code(‘lianda0707‘, ‘ld18513185949.07‘, ‘./img.jpg‘, 1004) # 模拟登录 url = ‘https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx‘ data = { "__VIEWSTATE":"FtrD1RIDtJDgCDOLII4PoI3KO8h6/WZp+uZrKxMZEhiTioqiAR9+e0f3i8Mvg+jCiSjYf9Et4rFcPgmghFuf7xRaS6K9nsJS4HFBCRKcUvPovmSc9fyYoTjUGZY=", "__VIEWSTATEGENERATOR":"C93BE1AE", "from":"http://so.gushiwen.org/user/collect.aspx", "email":"[email protected]", "pwd":"ld18513185949.07", "code":img_text, "denglu":"登录", } page_text = s.post(url=url, headers=headers, data=data).text print(page_text) with open(‘./gushiwen.html‘,‘w‘,encoding=‘utf8‘) as fp: fp.write(page_text)
2.selenium的使用
环境安装： pip install selenium 编码流程： 导包：from selenium import webdriver 实例化某一款浏览器对象 自制定自动化操作代码
from selenium import webdriver from time import sleep bro = webdriver.Chrome(executable_path=r‘F:爬虫+数据day03chromedriver_win32chromedriver.exe‘) bro.get(url=‘https://www.baidu.com/‘) sleep(2) text_input = bro.find_element_by_id(‘kw‘) text_input.send_keys("人民币") sleep(2) bro.find_element_by_id(‘su‘).click() sleep(3) # 获取当前页面源码数据 print(bro.page_source) bro.quit()
1 url = ‘https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action=‘ 2 3 bro = webdriver.Chrome(executable_path=r‘F:爬虫+数据day03chromedriver_win32chromedriver.exe‘) 4 bro.get(url) 5 sleep(3) 6 7 bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) 8 sleep(3) 9 bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) 10 sleep(3) 11 bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) 12 13 sleep(2) 14 page_text = bro.page_source 15 16 with open(‘./douban.html‘,‘w‘,encoding=‘utf8‘) as fp: 17 fp.write(page_text) 18 19 sleep(1) 20 bro.quit()
# 谷歌无头浏览器 from selenium.webdriver.chrome.options import Options chrome_options = Options() chrome_options.add_argument(‘--headless‘) chrome_options.add_argument(‘--disable-gpu‘) #获取豆瓣电影中更多电影详情数据 url = ‘https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action=‘ bro = webdriver.Chrome(executable_path=r‘F:爬虫+数据day03chromedriver_win32chromedriver.exe‘,chrome_options=chrome_options) #多加一个参数 bro.get(url) sleep(3) bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) sleep(3) bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) sleep(3) bro.execute_script(‘window.scrollTo(0,document.body.scrollHeight)‘) sleep(2) page_text = bro.page_source with open(‘./douban.html‘,‘w‘,encoding=‘utf-8‘) as fp: fp.write(page_text) print(page_text) sleep(1)
# qq空间 bro = webdriver.Chrome(executable_path=r‘F:爬虫+数据day03chromedriver_win32chromedriver.exe‘) url = ‘https://qzone.qq.com/‘ bro.get(url=url) sleep(2) # NoSuchElementException 这个错 # 定位到一个具体的iframe bro.switch_to.frame(‘login_frame‘) bro.find_element_by_id(‘switcher_plogin‘).click() sleep(2) bro.find_element_by_id(‘u‘).send_keys(‘460086804‘) bro.find_element_by_id(‘p‘).end_keys(‘shuo0127‘) bro.find_element_by_id(‘login_button‘).click() sleep(5) page_text = bro.page_source
3.线程的使用
# 爬取梨视频数据 import requests import re import random from lxml import etree from multiprocessing.dummy import Pool # 实例化一个线程池对象 pool = Pool(5) url = ‘https://www.pearvideo.com/category_1‘ headers = { ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36‘ } page_text = requests.get(url=url, headers=headers).text tree = etree.HTML(page_text) li_list = tree.xpath(‘//div[@id="listvideoList"]/ul/li‘) video_url_list = [] for li in li_list: detail_url = "https://www.pearvideo.com/" + li.xpath(‘./div/a/@href‘)[0] detail_page = requests.get(url=detail_url, headers=headers).text video_url = re.findall(‘srcUrl="(.*?)",vdoUrl‘,detail_page,re.S)[0] # 视频的url在script代码中，用“.mp4”查找出来的 video_url_list.append(video_url) video_data_list = pool.map(getVideoDate, video_url_list) pool.map(saveVideo, video_data_list) def getVideoDate(url): return requests.get(url=url,headers=headers).content def saveVideo(data): filename = str(random.randint(0,5000)) + ‘.mp4‘ with open(filename,‘wb‘) as fp: fp.write(data)
以上是关于爬虫入门的主要内容，如果未能解决你的问题，请参考以下文章