爬虫练习 | 爬取猫眼电影Top100

Posted 2021-01-25 404noofound

#coding=utf-8
_date_ = ‘2018/12/9 16:18‘
import requests
import re
import json
import time
def get_one_page(url):

    headers={
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/70.0.3538.110 Safari/537.36‘
    }
    reg=requests.get(url,headers=headers)
    if reg.status_code==200:
        return reg.text
    else:
        print(‘出错了‘)
def write_to_file(conments):
    with open(‘page.text‘,‘a‘,encoding=‘utf-8‘)as a:
        a.write(json.dumps(conments,ensure_ascii=False)+‘
‘)

def parse_one_page(html):
    r=re.compile(‘<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?"integer.*?>(.*?)</i>.*?"fraction.*?>(.*?)</i>.*?</dd>‘,re.S)
    items=re.findall(r,html)
    for item in items:
        yield {
            ‘index‘:item[0],
            ‘image‘:item[1],
            ‘name‘:item[2],
            ‘star‘:item[3].strip(),
            ‘time‘:item[4].strip(),
            ‘score‘:item[5]+item[6]
        }

if __name__ == ‘__main__‘:
    for i in range(0,10):
        url=‘https://maoyan.com/board/4?offset={}‘.format(i*10)
        html=get_one_page(url)
        content=parse_one_page(html)
        for i in content:
            write_to_file(i)
        time.sleep(1)