python—爬虫

Posted

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了python—爬虫相关的知识,希望对你有一定的参考价值。

1.1 介绍
通过过滤和分析HTML代码,实现对文件、图片等资源的获取,一般用到:
urllib和urllib2模块
正则表达式(re模块)
requests模块
Scrapy框架

urllib库:
1)获取web页面
2)在远程http服务器上验证
3)表单提交(GET和POST)
4)异常处理(urllib2.URLError)
5)非http协议通信(ftp)

获取页面信息:
urllib2.urlopen(url,data,timeout)
构造Request
reques = URLlib.Request(url,data,headers={})
response = urllib2.urlopen(request)
response.read()

1.2 爬取图片
百度贴吧示例链接: http://tieba.baidu.com/p/4229162765
1)从网页链接源代码中查找数据,用于分析和提取url
需下载的某张图片的url:
<img class="BDE_Image" src="http://imgsrc.baidu.com/forum/w%3D580/sign=d51025efb5fb43161a1f7a7210a54642/3887e950352ac65cf8452357fcf2b21193138a56.jpg" size="76453" width="400" height="600" style="cursor: url("http://tb2.bdstatic.com/tb/static-pb/img/cur_zin.cur";), pointer;">

2)脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#! /usr/bin/env python
import urllib,urllib2
import re
def getHtml(url):
page = urllib2.urlopen(url)
return page.read()

def getImage(html):
re_img = re.compile(r‘<img class="BDE_Image" src="(.?)".?‘) #“(.*?)”问号表示非贪婪模式,匹配到最接近的双引号”,而不加问号则匹配到最后
img_list = re_img.findall(html)
i = 1
for imgurl in img_list:
print imgurl
urllib.urlretrieve(imgurl,filename="%s.jpg" %i) #urllib.urlliburlretrieve下载,不指定文件名,则保持在当前目录
i = i+1

if name == "main":
url = "http://tieba.baidu.com/p/4229162765"
html = getHtml(url)
getImage(html)

运行结果:
1.png

1.3 爬取文本—获取作者、内容、点赞数
糗事百科热门段子示例链接:
第一页:https://www.qiushibaike.com/
第二页:https://www.qiushibaike.com/8hr/page/2/
第三页:https://www.qiushibaike.com/8hr/page/3/

1)随机选择一个段子,审查元素,获取:作者、内容、点赞个数url位置,用于定义正则表达式
作者位置<<<<<<
1.png

内容和点赞数位置<<<<<<
3.png

2)>>>>>>脚本版本一:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#!/usr/bin/env python
import urllib,urllib2
import re
page = 1
url = "https://www.qiushibaike.com/8hr/page/" +str(page)
headers = {"user-agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"} #访问正规的网站,一般需要user-agent
try:
request = urllib2.Request(url,headers=headers)
response = urllib2.urlopen(request)
html = response.read()

except urllib2.URLError,e:
if hasattr(e,"code"): #抛出异常时,e表示前面的错误类;判断该类中是否有code属性,有则打印出来
print e.code
if hasattr(e,"reason"):
print e.reason

re_page = re.compile(r‘<div class="author.?<a.?<img.?alt="(.?)">.?<div.?<span>(.?)</span>.?<i class="number">(\d+)</i>‘,re.S) #正则表达式,re.S表示点号可以代表换行符
items = re_page.findall(html)
for item in items:
for i in item:
print i

运行结果:
1.png

>>>>>>脚本版本二:
替换掉内容中网页换行符<br/>,然后去掉空格行,显示页数默认为第一页

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#!/usr/bin/env python
#coding:utf-8
import urllib,urllib2
import re

def getPage(page_num=1):
url = "https://www.qiushibaike.com/8hr/page/" +str(page_num)
headers = {"user-agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}
try:
request = urllib2.Request(url,headers=headers)
response = urllib2.urlopen(request)
html = response.read()
return html
except urllib2.URLError,e:
if hasattr(e,"code"):
print ("连接服务器失败,错误代码 %s" %e.code)
return None

    if hasattr(e,"reason"):
        print ("连接服务器失败,错误原因 %s" %e.reason)
        return None

def getPageCoent(page_num=1):
html = getPage(page_num)
re_page = re.compile(r‘<div class="author.?<a.?<img.?alt="(.?)">.?<div.?<span>(.?)</span>.?<i class="number">(\d+)</i>‘,re.S)
items = re_page.findall(html)
page_contents = [] #定义空列表,用于存放内容
replaceBR = re.compile(r‘<br/>‘)
for item in items:
content = item[1]
content = replaceBR.sub(‘\n‘,content) #替换换行符
page_contents.append([page_num,
item[0].strip(), #删掉空格
content.strip(),
item[2].strip()])
return page_contents

if name == "main":
page_content = getPageCoent(1)
for item in page_content:
for i in item:
print str(i) +"\n"

运行结果:
1.png

>>>>>脚本版本三:
实现交互式爬取
即每按一次enter键,显示一条段子,内容包括:页码、作者、段子内容、点赞数
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#!/usr/bin/env python
#coding:utf-8
import urllib,urllib2
import re
import sys

def getPage(page_num=1):
url = "https://www.qiushibaike.com/8hr/page/" +str(page_num)
headers = {"user-agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}

try:
    request = urllib2.Request(url,headers=headers)
    response = urllib2.urlopen(request)
    html = response.read()
    return html

except urllib2.URLError,e:
    if hasattr(e,"code"):
        print ("连接服务器失败,错误代码 %s" %e.code)
        return None

    if hasattr(e,"reason"):
        print ("连接服务器失败,错误原因 %s" %e.reason)
        return None

def getPageCoent(page_num=1):
html = getPage(page_num)
re_page = re.compile(r‘<div class="author.?<a.?<img.?alt="(.?)">.?<div.?<span>(.?)</span>.?<i class="number">(\d+)</i>‘,re.S)
items = re_page.findall(html)
page_contents = []
replaceBR = re.compile(r‘<br/>‘)
for item in items:
content = item[1]
content = replaceBR.sub(‘\n‘,content)
page_contents.append([page_num,
item[0].strip(),
content.strip(),
item[2].strip()])
return page_contents

def getOneStory(page_contents):
for story in page_contents:
input = raw_input()
if input == "q" or input == "Q":
sys.exit()
print "第%d页\t发布人:%s\t赞:%s\n%s\n" %(story[0],story[1],story[3],story[2])

if name == "main":
print "正在读取段子,按回车看新段子,按(Q|q)退出"
num = 1
page_contents = getPageCoent(num)
while True:
page_contents = getPageCoent(num)
getOneStory(page_contents)
num += 1

运行结果:
1.png

以上是关于python—爬虫的主要内容,如果未能解决你的问题,请参考以下文章

Python练习册 第 0013 题: 用 Python 写一个爬图片的程序,爬 这个链接里的日本妹子图片 :-),(http://tieba.baidu.com/p/2166231880)(代码片段

python爬虫学习笔记-M3U8流视频数据爬虫

爬虫遇到头疼的验证码?Python实战讲解弹窗处理和验证码识别

python网络爬虫

Python 利用爬虫爬取网页内容 (div节点的疑惑)

为啥我的python爬虫界面与博主不一样