Stage6--Python简单爬虫
Posted lxq_xsyu
tags:
篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了Stage6--Python简单爬虫相关的知识,希望对你有一定的参考价值。
正则表达式简单介绍
正则表达式(regular expression)描述了一种字符串匹配的模式,可以用来检查一个串是否含有某种子串、将匹配的子串做替换或者从某个串中取出符合某个条件的子串等。
字符 [a-z]
数字 [0-9] 或 \\b
* 匹配前面的子表达式零次或多次
+ 匹配前面的子表达式一次或多次
? 匹配前面的子表达式零次或一次
一个简单爬虫例子
import re
import urllib.request
url = "http://mall.csdn.net/coin"
savePath = "G:/QQData/"
def gethtml(url):
page = urllib.request.urlopen(url)
html = page.read().decode('utf-8')
return html
def getImg(html):
reg = r'http:\\/\\/img.bss.csdn.net\\/[0-9]+\\.jpg'
imgre = re.compile(reg)
imglist = re.findall(imgre, html)
return imglist;
def saveImg(url):
conn = urllib.request.urlopen(url)
file = open(savePath + getFileName(url), 'wb')
file.write(conn.read())
file.close()
return
def getFileName(url):
reg = r'[a-z0-9]+\\.jpg'
matchObj = re.search(reg, url)
if matchObj:
return matchObj.group()
return
html = str(getHtml(url));
imgurls = getImg(html);
for imgurl in imgurls:
print(imgurl)
saveImg(imgurl)
这个例子很简单,里面的正则简单的到没有,是爬取CSDN U币商城的图片,很容易看懂。
可以自定义正则和Url的爬虫
from tkinter import *
import re
import urllib.request
savePath = "G:/QQData/"
class ControllPanel(Frame):
default_url = "http://www.xingyongshe.com/man/ddfeijibei"
default_regix = r'<img src="(.+)&w=228&q=90&type=jpg'
__status_label = None #私有属性
url_input = None
regix_input = None
def __init__(self, master):
Frame.__init__(self, master, width=360, height=260)
self.master = master
self.pack()
self.add_urlarea()
self.add_regixarea()
self.add_buttonarea()
self.add_statusbar()
def add_urlarea(self):
urlAreaFrame = Frame(self)
urlAreaFrame.pack(pady=20)
label = Label(urlAreaFrame, text="网址:")
label.pack(side=LEFT)
self.url_input = Entry(urlAreaFrame, width=40)
self.url_input.insert(0, ControllPanel.default_url)
self.url_input.pack(side=RIGHT)
def add_regixarea(self):
regixAreaFrame = Frame(self)
regixAreaFrame.pack()
label = Label(regixAreaFrame, text="正则:")
label.pack(side=LEFT)
self.regix_input = Entry(regixAreaFrame, width=40)
self.regix_input.insert(0, self.default_regix) #注意这里的访问方式和上面方式不同
self.regix_input.pack(side=RIGHT)
def add_buttonarea(self):
buttonAreaFrame = Frame(self)
buttonAreaFrame.pack(pady=40)
verify_button = Button(buttonAreaFrame, text="验证正则", command=self.verifyRegix)
verify_button.pack(padx=50, side=LEFT)
start_button = Button(buttonAreaFrame, text="开始爬取", command=self.startRun)
start_button.pack(padx=50, side=RIGHT)
def verifyRegix(self):
if self.url_input == None or self.regix_input == None:
return
url = self.url_input.get()
regix = self.regix_input.get()
runfunction = RunFunction(url, regix)
imgurls = runfunction.verifyRegix()
if len(imgurls) > 0:
self.setStatus("有%d张图片可以爬取", len(imgurls))
for imgurl in imgurls:
print(imgurl)
return imgurls
def startRun(self):
imgurls = self.verifyRegix()
runfunction = RunFunction(None, None)
size = runfunction.runTask(imgurls)
self.setStatus("爬取了%d张图片放在了" + savePath, size)
def add_statusbar(self):
statusBarFrame = Frame(self.master)
statusBarFrame.pack(side=BOTTOM, fill=X)
self.status_label = Label(statusBarFrame, bd=1, relief=SUNKEN, anchor=W)
self.status_label.pack(fill=X)
def setStatus(self, format, *args):
if self.status_label == None:
return
self.status_label.config(text=format % args)
self.status_label.update_idletasks()
return
def clearStatus(self):
self.status_label.config(text="")
self.status_label.update_idletasks()
class RunFunction:
def __init__(self, url, regix):
self.url = url
self.regix = regix
def getHtml(self, url):
page = urllib.request.urlopen(url)
html = page.read().decode('utf-8')
return html
def getImg(self, html, reg):
imgre = re.compile(reg)
imglist = re.findall(imgre, html)
return imglist;
def saveImg(self, url):
conn = urllib.request.urlopen(url)
file = open(savePath + self.getFileName(url), 'wb')
file.write(conn.read())
file.close()
return
def getFileName(self, url):
reg = r'[a-z0-9]+\\.jpg'
matchObj = re.search(reg, url)
if matchObj:
return matchObj.group()
return
def verifyRegix(self):
html = str(self.getHtml(self.url))
imgurls = self.getImg(html, self.regix)
return imgurls
def runTask(self):
imgurls = self.verifyRegix(self.url, self.regix)
for imgurl in imgurls:
self.saveImg(imgurl)
return len(imgurls)
def runTask(self, imgurls):
for imgurl in imgurls:
self.saveImg(imgurl)
return len(imgurls)
root = Tk()
root.title("爬虫管理窗口")
#让窗口居中显示
scnWidth, scnHeight = root.maxsize()
tmpcnf = '%dx%d+%d+%d'%(308, 101, (scnWidth-308)/2, (scnHeight-101)/2)
root.geometry(tmpcnf)
root.maxsize(600, 300)
root.minsize(360, 220)
#root.resizable(False, False) #让窗口尺寸不变
controllPanel = ControllPanel(root)
controllPanel.setStatus("等待爬取……")
root.mainloop()
root.destroy()
以上是关于Stage6--Python简单爬虫的主要内容,如果未能解决你的问题,请参考以下文章