2018/7/21 Python 爬虫学习

Posted 2020-12-24 bqwzx

2018/7/21，这几天整理出来的一些Python 爬虫学习代码。

import urllib2

response = urllib2.urlopen("http://baidu.com")

html = response.read()

print html

进一步，可以request

import urllib2

req = urllib2.Request("http://www.baidu.com")

response = urllib2.urlopen(req)

html = response.read()

print html

伪装浏览器

import urllib2
url = "http://www.baidu.com"
user_agent = "Mozilla/5.0(compatible;MSTE 9.0;Windows NT 6.1;Trident/5.0;"
headers = {"User-Agent‘:user_agent}
req = urllib2.Request(rul,headers = headers)
response = urllib2.urlopen(req)
the_page = response.read()
print the_page

代码：输入输出网页

# _*_ coding:utf-8 _*_
import urllib2

def load_page(url):
user_agent = "Mozilla/5.0 (compatible;MSTE 9.0;Windows NT 6.1;Trident/5.0;"
headers = {"User-Agent":user_agent}
req = urllib2.Request(url,headers = headerss)
response = urllib2.urlopen(req)
html = response.read()
return html

def tieba_spider(url,begin_page,end_page):
"""
贴吧爬虫的方法
"""
for i in range(begin_page,end_page +1):
pn = 50 * (i-1)
my_url = url + str(pn)
html = load_page(my_url)
print "##################第%页########################" %(i)
print html
print "###############################################"

if __name__ == "__main__":
url = raw_input("请输入贴吧的url地址")
begin_page = int(raw_input("请输入起始页码"))
end_page = int(raw_input("请输入终止页码"))

tieba_spider(url,begin_page,end_page)

代码：输入输出保存网页

# _*_ coding:utf-8 _*_
import urllib2

def load_page(url):
user_agent = "Mozilla/5.0 (compatible;MSTE 9.0;Windows NT 6.1;Trident/5.0;"
headers = {"User-Agent":user_agent}
req = urllib2.Request(url,headers = headers)
response = urllib2.urlopen(req)
html = response.read()
return html

def writee_to_file(file_name,txt):
"""将txt文本存入到file_name文件中
"""
print "正在存储文件" +filr_name
f = oprn(file_name,‘w‘)
f = write(txt)
f.close(0

def tieba_spider(url,begin_page,end_page):
"""
贴吧爬虫的方法
"""
for i in range(begin_page,end_page + 1):
pn = 50 * (i-1)
my_url = url + srt(pn)
html = load_page(my_url)

filr_name = str(i) + ".html"
write_to_file(file_name,html)

if __name__ == "__main__":
url = raw_input("请输入贴吧的url地址")
begin_page = int(raw_input("请输入起始页码"))
end_page = int(raw_input("请输入终止页码"))

tieba_spider(url,begin_page,end_page)