Python简易爬虫

Posted 2020-09-17

#  coding: utf-8
import urllib
import urllib2
import re
import os

if __name__==‘__main__‘:
    print "抓取开始..."
    j = 1
    for i in range(1,35):
        url=‘http://www.qiushibaike.com/8hr/page/‘+str(i)+‘/?s=4981088‘
        header={‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36‘}
        try:
            request=urllib2.Request(url=url,headers=header)
            response=urllib2.urlopen(request)
            content= response.read()
        except urllib2.HTTPError as e:
            print e
            exit()
        except urllib2.URLError as e:
            print e
            exit()
        pattern=re.compile(‘<div class="content">.*?<span>(.*?)</span>.*?</div>‘,re.S)
        items=re.findall(pattern,content)
        path="qiubai"
        if not os.path.exists(path):
            os.makedirs(path)
        for item in items:
            file_path=path+"/"+str(j)+‘.txt‘
            f=open(file_path,‘w‘)
            item=item.replace(‘<br/>‘,‘\n‘)
            f.write(item)
            f.close()
            j=j+1
    print "内容抓取完成..."