（爬虫实例1）构建电影天堂

Posted 2021-08-11 是璇子鸭

对应页面

提取代码

import requests
from lxml import etree

headers= {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    # 'Host': 'www.dytt8.net',
    # 'Referer':'https://www.dytt8.net/',
}

BASE_DOMAIN = 'https://www.dytt8.net'

def pasrse_page(url):
    '''
    发送请求 获取详情
    :param url:
    :return:
    '''
    movie = {} #存放一部电影的详细信息
    res = requests.get(url, headers=headers)
    data = res.content.decode('gbk')
    html = etree.HTML(data)
    title = html.xpath('/html/body/div[1]/div/div[3]/div[3]/div[1]/div[2]/div[1]/h1/font//text()')[0]
    movie['title'] = title
    Zoom = html.xpath("//div[@id='Zoom']")[0]
    cover = Zoom.xpath(".//img/@src")
    movie['cover'] = cover
    infos = Zoom.xpath(".//text()")
    for index,info in enumerate(infos):
        # print(index,info)
        if info.startswith("◎年　　代"):
            year = info.replace("◎年　　代","").strip() #strip去除空格与换行
            movie['year'] = year

        elif info.startswith("◎产　　地　俄罗斯"):
            country = info.replace("◎产　　地","").strip()
            movie['country'] = country

        elif info.startswith("◎类　　别"):
            category = info.replace("◎类　　别","").strip()
            movie['category'] = category

        elif info.startswith("◎语　　言"):
            language = info.replace("◎语　　言","").strip()
            #print(language)
            movie['language'] = language

        elif info.startswith("◎字　　幕"):
            trans = info.replace("◎字　　幕","").strip()
            movie['trans'] = trans

        elif info.startswith("◎上映日期"):
            date = info.replace("◎上映日期","").strip()
            movie['date'] = date

        elif info.startswith("◎片　　长"):
            time = info.replace("◎片　　长","").strip()
            movie['time'] = time

        elif info.startswith("◎导　　演"):
            director = info.replace("◎导　　演", "").strip()
            movie['director'] = director

        elif info.startswith("◎主　　演"):
            info = info.replace("◎主　　演","").strip()
            actors = [info]
            for x in range(index+1, len(infos)):
                actor = infos[x].strip()
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            movie['actors'] = actors
            # print(actors)

        elif info.startswith("◎简　　介"):
            info = info.replace("◎简　　介","").strip()
            for x in range(index+1, len(infos)):
                profile = infos[x].strip()
                if profile.startswith("磁"):
                    break
                movie['profile'] = profile
                # print(profile)

    download_url = Zoom.xpath('.//a/@href')[0]
    movie['download_url'] = download_url
    return movie

def get_detail_urls(url):
    '''
    获取这一页中每部电影的详情链接
    :param url:
    :return: 返回一个列表 包含每个电影的详情链接
    '''

    res = requests.get(url, headers = headers)
    #print(res.content.decode('gbk'))
    text = res.text
    html = etree.HTML(text)
    detail_urls = html.xpath("//table[@class='tbspan']//a/@href")
    detail_urls = list(map(lambda url:BASE_DOMAIN + url, detail_urls))
    return detail_urls

for x in range(1,51):
    url = f"https://www.dytt8.net/html/gndy/dyzz/list_23_{x}.html"
    detail_urls = get_detail_urls(url)
    movies = []
    for detail_url in detail_urls:
        movie = pasrse_page(detail_url)
        movies.append(movie)
        print(movie)

运行结果：