Python爬虫：爬取糗事百科

Posted 2020-07-08 小星星学数据

网上看到的教程，但是是用正则表达式写的，并不能运行，后面我就用xpath改了，然后重新写了逻辑，并且使用了双线程，也算是原创了吧
#!/usr/bin/python
# -*- encoding:utf-8 -*-


from lxml import etree
from multiprocessing.dummy import Pool as ThreadPool
import requests
import sys
#编码
reload(sys)
sys.setdefaultencoding(‘utf-8‘)

#定义输出函数
def towrite(contentdict):
    f.writelines(u‘作者:‘ + contentdict[‘author‘] + ‘\n‘)
    f.writelines(u‘内容:‘ + contentdict[‘content‘] + ‘\n‘)
    f.writelines(u‘好笑:‘ + contentdict[‘vote‘] + ‘\n‘)
    f.writelines(u‘评论:‘ + contentdict[‘span‘] + ‘\n\n‘)

def spider(url):
    #得到页面代码
    user_agent = ‘Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)‘
    headers = { ‘User-Agent‘ : user_agent }
    html = requests.get(url,headers=headers)

    #获取内容
    selector = etree.HTML(html.text)
    content_field = selector.xpath(‘//*[@id="content-left"]/div[@class="article block untagged mb15"]‘)
    item={}
    for i in range(len(content_field)):
        #作者
        author_f= content_field[i].xpath(‘div[@class="author clearfix"]‘)[0]
        author=author_f.xpath(‘string(.)‘).replace(‘\n‘,‘‘).replace(‘ ‘,‘‘)
        #内容
        content_f=content_field[i].xpath(‘div[@class="content"]/text()‘)
        content=‘‘
        for n in range(len(content_f)):
            content_temp=content_f[n].replace(‘\n‘,‘‘).replace(‘ ‘,‘‘).replace(‘\t‘,‘‘)
            content+=str(content_temp)

        #好笑
        vote=‘‘
        vote_temp= content_field[i].xpath(‘div[@class="stats"]/span[@class="stats-vote"]/i/text()‘)[0]
        vote+=str(vote_temp)

        #评论,如果评论为空，则不会显示i节点
        span=‘‘
        span_temp_l= content_field[i].xpath(‘div[@class="stats"]/span[@class="stats-comments"]/a/i/text()‘)
        span_temp=[]
        if len(span_temp_l)>0:
            span_temp=span_temp_l[0]
        else:
            span_temp=‘0‘
        span+=str(span_temp)

        item[‘author‘] = author
        item[‘content‘] = content
        item[‘vote‘] = vote
        item[‘span‘] = span

        towrite(item)


if __name__ == ‘__main__‘:
    pool = ThreadPool(4)
    f = open(‘content.txt‘,‘a‘)
    url = []
    for i in range(1,36):
        newpage = ‘http://www.qiushibaike.com/hot/page/‘ + str(i)
        url.append(newpage)

    results = pool.map(spider, url)
    pool.close()
    pool.join()
    f.close()