urllib：简单的贴吧页面爬取代码

Posted 2022-06-13 wshr210

from urllib import request
import time
import urllib
import urllib.parse

#根据url发送请求，获取服务器响应文件
def loadPage(url,filename):
    print(‘正在下载‘ + filename)
    headers = 
        ‘User - Agent‘: ‘Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(Khtml, like Gecko) Chrome / 65.0.3325.181 Safari / 537.36‘
    
    req = urllib.request.Request(url,headers=headers)
    return urllib.request.urlopen(req).read()

#将html内容写入本地
def writePage(html,filename):
    print(‘正在保存‘ + filename)
    with open(filename,‘wb‘) as f:
        f.write(html)
    print(‘-------------------------------‘)



# http://tieba.baidu.com/f?kw=python&fr=ala0&tpl=5    第一页
# http://tieba.baidu.com/f?kw=python&ie=utf-8&pn=0      按照规律这个和上面的第一页url是一样的

# http://tieba.baidu.com/f?kw=python&ie=utf-8&pn=50     第二页

# http://tieba.baidu.com/f?kw=python&ie=utf-8&pn=100   第三页

# http://tieba.baidu.com/f?kw=python&ie=utf-8&pn=150    第四页

# http://tieba.baidu.com/f?kw=python&ie=utf-8&pn=200    第五页



#处理每个页面的url
def tiebaSpider(url,beginPage,endPage):
    for page in range(beginPage,endPage+1):
        pn = (page - 1) * 50
        filename=‘d:/yemian/第‘+str(page) + ‘页.html‘
        fullurl = url + ‘&pn-‘ + str(pn)
        html = loadPage(fullurl,filename)

        writePage(html,filename)


if __name__ == ‘__main__‘:
    kw = input(‘请输入需要爬取页面的贴吧名:‘)
    beginPage = int(input(‘请输入起始页:‘))
    endPage = int(input(‘请输入结束页:‘))

    url = ‘http://tieba.baidu.com/f?‘
    key = urllib.parse.urlencode(‘kw‘:kw)
    fullurl = url + key
    tiebaSpider(fullurl,beginPage,endPage)

print(‘谢谢使用‘)
time.sleep(10)