Python-爬虫-猫眼T100

Posted 2021-02-03 天宝老爹

目标站点需求分析

涉及的库

from multiprocessing import Pool
from requests.exceptions import RequestException
import requests
import json,csv,time,re

获取单页源码

def get_one_page(url):
    \'\'\'获取单页源码\'\'\'
    try:
        headers = {
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
        }
        res = requests.get(url, headers=headers)
        time.sleep(1)
        # 判断响应是否成功
        if res.status_code == 200:
            # print(res.text)
            return res.text
        return (\'status_code error\')
    except RequestException:
        return (\'RequestException error\'

解析单页源码

def parse_one_page(html):
    \'\'\'解析单页源码\'\'\'
    pattern = re.compile(\'<dd>.*?board-index.*?>(\\d+)</i>.*?name"><a.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime\'
                         + \'.*?>(.*?)</p>.*?score.*?integer">(.*?)</i>.*?>(.*?)</i>.*?</dd>\',re.S)
    items = re.findall(pattern,html)
    #循环提取信息
    for item in  items:
        yield {
            \'rank\' :item[0],
            \'name\':item[1],
            # \'actor\':item[2].strip()[3:] if len(item[2])>3 else \'\',  #判断是否大于3个字符
            # \'time\' :item[3].strip()[5:] if len(item[3])>5 else \'\',
            \'actor\':item[2].strip()[3:],
            \'time\' :item[3].strip()[5:15],
            \'score\':item[4] + item[5]
        }

保存到文件中

def write_to_textfile(content):
    \'\'\'写入text\'\'\'
    with open("MovieResult.text",\'a\',encoding=\'utf-8\') as f:
        #利用json.dumps()将字典序列化,并将ensure_ascii设置为False,从而显示中文.+换行
        f.write(json.dumps(content,ensure_ascii=False) + "\\n")
        f.close()

def write_to_csvField(fieldnames):
    \'\'\'写入csv表头\'\'\'
    with open("MovieResult.csv", \'a\', encoding=\'gb18030\', newline=\'\') as f:
        #将字段名传给Dictwriter来初始化一个字典写入对象
        writer = csv.DictWriter(f,fieldnames=fieldnames)
        #调用writeheader方法写入字段名
        writer.writeheader()

def write_to_csvRows(content,fieldnames):
    \'\'\'写入csv内容\'\'\'
    with open("MovieResult.csv",\'a\',encoding=\'gb18030\',newline=\'\') as f:
        #将字段名传给Dictwriter来初始化一个字典写入对象
        writer = csv.DictWriter(f,fieldnames=fieldnames)
        writer.writerows(content)
        f.close()

开启多进程获取多个页面抓取

# 将字段名传入列表
fieldnames = ["rank", "name", "actor", "time", "score"]

def task(offset):
    url = "http://maoyan.com/board/4?offset={0}".format(offset)
    html = get_one_page(url)
    rows = []
    for item in parse_one_page(html):
        # write_to_textfile(item)
        rows.append(item)
    # 写入csv内容
    write_to_csvRows(rows,fieldnames)

if __name__ == \'__main__\':
    #写入csv表头
    write_to_csvField(fieldnames)
    #map方法会把每个元素当做函数的参数,,在进程池中创建多进程.循环写入10页传参
    pool = Pool()
    pool.map(task,[i*10 for i in range(10)])