PySpider 抓取猫眼电影TOP100

Posted 小怪兽的技术博客

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了PySpider 抓取猫眼电影TOP100相关的知识,希望对你有一定的参考价值。

"""
抓取猫眼电影TOP100
"""

import re
import time
import requests

from bs4 import BeautifulSoup


class SpiderMaoyan(object):

    def __init__(self):
        # 通过分析URL可以发现, 猫眼电影TOP100页面是通过 offset + 10 来分页的
        self.url = "http://maoyan.com/board/4?offset={0}"
        # 设置一下UA, 否则有可能提示你访问被禁止了
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) "
                          "Chrome/66.0.3359.139 Safari/537.36"
        }
        # 定义需要提取的内容字段
        self.fields = ("id", "name", "movieUrl", "imgUrl", "star", "releaseTime", "score")

    def handler(self, offset=0):

        while offset < 100:
            response = requests.get(self.url.format(offset), headers=self.headers)
            if response.status_code == 200:
                print("INFO -> Current URL: <%s>" % response.url)
                # 编码处理一下, 不然有可能中文显示乱码
                r_html = response.text.encode(response.encoding).decode("utf-8")
                # 构建一个 BeautifulSoup 对象, 用于后续的标签、内容提取
                soup = BeautifulSoup(r_html, "html5lib")
                # 继续分析网页源代码, 发现每部影片都存在 <dd></dd> 标签中
                tags = soup.select("dd")
                # 提取内容
                for tag in tags:
                    # id、name、movieUrl
                    obj = tag.find("p", class_="name").select_one("a")
                    _id = re.search(r"movieId:(\d+)", obj.get("data-val")).group(1)
                    _name = obj.string
                    _movieUrl = "http://maoyan.com" + obj.get("href")
                    # img
                    # Tips: 可以将图片地址后面的分辨率去掉, 保存高清大图地址 .split("@")[0]
                    _imgUrl = tag.find("img", class_="board-img").get("data-src")
                    # star
                    # Tips: 中文标点
                    _star = tag.find("p", class_="star").string.strip().split("")[-1]
                    # releaseTime
                    # Tips: 中文标点
                    _releaseTime = tag.find("p", class_="releasetime").string.split("")[-1]
                    # score
                    _score = tag.find("p", class_="score").get_text()

                    # 接下来就可以将数据写入存储了
                    # Tips: 这种 SQL 生成方式有必要验证 key/val 是否成对出现
                    print(
                        "INSERT INTO TABLE_NAME (%s) VALUE %s;" % (
                            ", ".join(self.fields), tuple([_id, _name, _movieUrl, _imgUrl, _star, _releaseTime, _score])
                        )
                    )

                # 偏移量自增
                offset += 10
                # 有必要停顿一下
                time.sleep(.9)
            else:
                print(response.reason)
                exit(999)


if __name__ == "__main__":

    spider = SpiderMaoyan()
    spider.handler()

 执行效果如下:

INFO -> Current URL: <http://maoyan.com/board/4?offset=0>
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (1203, 霸王别姬, http://maoyan.com/films/1203, http://p1.meituan.net/movie/[email protected]_220h_1e_1c, 张国荣,张丰毅,巩俐, 1993-01-01(中国香港), 9.6);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (1297, 肖申克的救赎, http://maoyan.com/films/1297, http://p0.meituan.net/movie/[email protected]_220h_1e_1c, 蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿, 1994-10-14(美国), 9.5);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (2641, 罗马假日, http://maoyan.com/films/2641, http://p0.meituan.net/movie/23/[email protected]_220h_1e_1c, 格利高利·派克,奥黛丽·赫本,埃迪·艾伯特, 1953-09-02(美国), 9.1);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (4055, 这个杀手不太冷, http://maoyan.com/films/4055, http://p0.meituan.net/movie/[email protected]_220h_1e_1c, 让·雷诺,加里·奥德曼,娜塔莉·波特曼, 1994-09-14(法国), 9.5);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (1247, 教父, http://maoyan.com/films/1247, http://p0.meituan.net/movie/92/[email protected]_220h_1e_1c, 马龙·白兰度,阿尔·帕西诺,詹姆斯·凯恩, 1972-03-24(美国), 9.3);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (267, 泰坦尼克号, http://maoyan.com/films/267, http://p0.meituan.net/movie/11/[email protected]_220h_1e_1c, 莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩, 1998-04-03, 9.5);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (123, 龙猫, http://maoyan.com/films/123, http://p0.meituan.net/movie/[email protected]_220h_1e_1c, 日高法子,坂本千夏,糸井重里, 1988-04-16(日本), 9.2);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (837, 唐伯虎点秋香, http://maoyan.com/films/837, http://p0.meituan.net/movie/62/[email protected]_220h_1e_1c, 周星驰,巩俐,郑佩佩, 1993-07-01(中国香港), 9.2);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (2760, 魂断蓝桥, http://maoyan.com/films/2760, http://p1.meituan.net/movie/[email protected]_220h_1e_1c, 费雯·丽,罗伯特·泰勒,露塞尔·沃特森, 1940-05-17(美国), 9.2);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (1212, 千与千寻, http://maoyan.com/films/1212, http://p0.meituan.net/movie/[email protected]_220h_1e_1c, 柊瑠美,入野自由,夏木真理, 2001-07-20(日本), 9.3);
INFO -> Current URL: <http://maoyan.com/board/4?offset=10>
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (7431, 乱世佳人, http://maoyan.com/films/7431, http://p0.meituan.net/movie/13/[email protected]_220h_1e_1c, 费雯·丽,克拉克·盖博,奥利维娅·德哈维兰, 1939-12-15(美国), 9.1);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (9025, 喜剧之王, http://maoyan.com/films/9025, http://p0.meituan.net/movie/59/[email protected]_220h_1e_1c, 周星驰,莫文蔚,张柏芝, 1999-02-13(中国香港), 9.2);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (1228, 天空之城, http://maoyan.com/films/1228, http://p0.meituan.net/movie/24/[email protected]_220h_1e_1c, 寺田农,鹫尾真知子,龟山助清, 1992, 9.1);
INSERT INTO TABLE_NAME (id, name, movieUrl, imgUrl, star, releaseTime, score) VALUE (14556, 大闹天宫, http://maoyan.com/films/14556, http://p0.meituan.net/movie/[email protected]_220h_1e_1c, 邱岳峰,毕克,富润生, 1965-12-31, 9.0);

 

以上是关于PySpider 抓取猫眼电影TOP100的主要内容,如果未能解决你的问题,请参考以下文章

反爬虫2-python3.6 正则表达式抓取猫眼电影TOP100

网站爬取-案例一:猫眼电影TOP100

00_抓取猫眼电影排行TOP100

Python爬虫编程思想(37):项目实战:抓取猫眼电影Top100榜单

Requests+正则表达式抓取猫眼电影TOP100

Requwsts+正则表达式爬取猫眼电影Top100