实现视频爬取, 自动评论, 自动点赞

Posted 2022-08-31 lanxiaofang

一、[知识点]:

   动态数据抓包

   requests发送请求

   json数据解析

二、[开发环境]:

   python 3.8               运行代码

   pycharm 2021.2           辅助敲代码

   requests                 pip install requests

三、爬虫案例:

   采集快手短视频网站视频

   分析数据来源

​​    https://www.kuaishou.com/graphql​​

四、实现代码:

   1. 发送请求

   2. 获取数据

   3. 解析数据

   4. 保存数据

爬虫:

   模拟成 浏览器 向 服务器 发送请求

五、完整代码

import requests     # 发送请求 第三方模块
import re

headers = 
    content-type: application/json,
    # 用户信息
    Cookie: kpf=PC_WEB; kpn=KUAISHOU_VISION; clientid=3; did=web_d3f9d8c2cbebafd126b80eb0b1c13360; client_key=65890b29; didv=1658130458000; userId=270932146; kuaishou.server.web_st=ChZrdWFpc2hvdS5zZXJ2ZXIud2ViLnN0EqABCj1Pe61TcGTRmOxDP2F7J-5buR1I6zTbr2o8VylTwBIilBXkjnTbXau3z8OK1r-i0YIefozg8oheW-VO5_33SX0PmlNy5A8bmqSsJXZocyw3CusEfPPuVrgD6zZlzHSqW-M7GKTSptfCJ6of43qs700fYxwy-yrx13---JA62jliXOadl2OOT9f_A7W7DdIhT8rMQtFFdodh_frGf3CyBhoSoJCKbxHIWXjzVWap_gGna5KjIiB6FJHOKt3vnbSSWhl2W0DWrtjoA1X_lW9zlGlRaYHPkSgFMAE; kuaishou.server.web_ph=7ee6499c7437971b1182aa3bb1ba1c645b9f,
    # 域名
    Host: www.kuaishou.com,
    Origin: https://www.kuaishou.com,
    # 防盗链
    Referer: https://www.kuaishou.com/profile/3xsxdmstbwbx4ba,
    # 浏览器基本信息
    User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/104.0.0.0 Safari/537.36,

url = https://www.kuaishou.com/graphql

def get_page(pcursor):
    json = 
        operationName: "visionProfilePhotoList",
        query: "fragment photoContent on PhotoEntity \\n  id\\n  duration\\n  caption\\n  likeCount\\n  viewCount\\n  realLikeCount\\n  coverUrl\\n  photoUrl\\n  photoH265Url\\n  manifest\\n  manifestH265\\n  videoResource\\n  coverUrls \\n    url\\n    __typename\\n  \\n  timestamp\\n  expTag\\n  animatedCoverUrl\\n  distance\\n  videoRatio\\n  liked\\n  stereoType\\n  profileUserTopPhoto\\n  __typename\\n\\n\\nfragment feedContent on Feed \\n  type\\n  author \\n    id\\n    name\\n    headerUrl\\n    following\\n    headerUrls \\n      url\\n      __typename\\n    \\n    __typename\\n  \\n  photo \\n    ...photoContent\\n    __typename\\n  \\n  canAddComment\\n  llsid\\n  status\\n  currentPcursor\\n  __typename\\n\\n\\nquery visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) \\n  visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) \\n    result\\n    llsid\\n    webPageArea\\n    feeds \\n      ...feedContent\\n      __typename\\n    \\n    hostName\\n    pcursor\\n    __typename\\n  \\n\\n",
        variables: userId: "3xhv7zhkfr3rqag", pcursor: pcursor, page: "profile"
    
    # 1. 发送请求
    response = requests.post(url=url, headers=headers, json=json)
    # 2. 获取数据
    json_data = response.json()
    # 3. 解析数据
    feeds = json_data[data][visionProfilePhotoList][feeds]
    pcursor = json_data[data][visionProfilePhotoList][pcursor]
    for feed in feeds:
        caption = feed[photo][caption]
        photoUrl = feed[photo][photoUrl]
        print(caption, photoUrl)
        photoAuthorId = feed[author][id]
        photoId = feed[photo][id]
        json_1 = 
            operationName: "visionVideoLike",
            query: "mutation visionVideoLike($photoId: String, $photoAuthorId: String, $cancel: Int, $expTag: String) \\n  visionVideoLike(photoId: $photoId, photoAuthorId: $photoAuthorId, cancel: $cancel, expTag: $expTag) \\n    result\\n    __typename\\n  \\n\\n",
            variables: 
                cancel: 0,
                expTag: "1_i/2005282647926093489_xpcwebprofilexxnull0",
                photoAuthorId: photoAuthorId,
                photoId: photoId
            
        
        requests.post(url=url, headers=headers, json=json_1)
        # caption = re.sub([\\\\/:"<>|*\\\\n], , caption)
        # # 4. 保存数据
        # video_data = requests.get(photoUrl).content
        # with open(fvideo/caption.mp4, mode=wb) as f:
        #     f.write(video_data)
    if pcursor == no_more:
        return 0
    get_page(pcursor)

get_page("")