python_day06(ip代理池)
Posted 窃语
tags:
篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了python_day06(ip代理池)相关的知识,希望对你有一定的参考价值。
from urllib.request import Request, ProxyHandler from urllib.request import build_opener from bs4 import BeautifulSoup import MySQLdb; import redis from urllib.request import urlopen from lxml import etree from lxml import etree import re; urlfront = "http://www.xicidaili.com" url = "http://www.xicidaili.com/nn/1" result = redis.Redis(host=‘127.0.0.1‘, port=6379,db=0) # def spider_IP(url): # 获取整个页面 def get_allcode(url): # 设置代理IP proxy = {‘https‘: ‘110.73.0.45:8123‘} proxy_support = ProxyHandler(proxy); opener = build_opener(proxy_support) # 设置访问http协议头,模拟浏览器 opener.addheaders = [ (‘User-agent‘, ‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6‘)] r = opener.open(url) html = r.read().decode("UTF-8"); # print(html) return str(html) # lxml 方式 获取Ip def find_ip(s): # s = get_allcode(url); selector = etree.HTML(s); links = selector.xpath(‘//tr[@class="odd"]/td/text()|//tr[@class=""]/td/text()‘); ip=[] port=[] for link in links: # print(link) if ‘-‘ in link: # print() pass elif link.isdigit(): port.append(link) # f.write(link + ‘\n‘); elif ‘.‘ in link: ip.append(link) # f.write(link + ‘:‘); # 用redis 的 llist存 ip for i in range(len(ip)): # print(ip[i]+":"+port[i]) ips=ip[i] + ":" + port[i] result.lpush(‘mylist‘,ips) def get_next_page(s): selecter = etree.HTML(s); link = selecter.xpath(‘//div[@class="pagination"]/a[@class="next_page"]/@href‘); for i in link: if i == None: return None; return urlfront + i def get_allcode_ip(url,ip): # 设置代理IP try: ip=str(ip, encoding="utf-8")# bytes与str相互转换 timeout=5 proxy = {‘http‘:ip} proxy_support = ProxyHandler(proxy); opener = build_opener(proxy_support) # 设置访问http协议头,模拟浏览器 opener.addheaders = [ (‘User-agent‘, ‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6‘)] # 加运行超时 r = opener.open(url,None,timeout) html = r.read().decode("UTF-8"); print(‘+++++++++++++++‘) # 将可用Ip放到redis的useable_ip中 result.lpush(‘usable_ip‘,ip) print(ip) print(‘+++++++++++++++‘) except Exception as err: print(err) while 1: print(url) s=get_allcode(url); url=get_next_page(s) print(url) if url==None: break find_ip(s) while 1: ip = result.lpop(‘mylist‘) print(ip) if ip == None: break get_allcode_ip(url, ip)
以上是关于python_day06(ip代理池)的主要内容,如果未能解决你的问题,请参考以下文章