爬取肯德基某地区所有门店信息

Posted 2020-12-12 sly27

import requests
import json


def get_response(keyword, num):
    post_data = {
        ‘cname‘: ‘‘,
        ‘pid‘: ‘‘,
        ‘keyword‘: keyword,
        ‘pageIndex‘: ‘1‘,
        ‘pageSize‘: num
    }
    response = requests.post(url=url, data=post_data, headers=headers)
    return response


dict_data = None
if __name__ == ‘__main__‘:
    keyword = input(‘请输入爬取的城市名：‘)
    num = 1
    i = 0
    url = ‘http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword‘
    # UA伪装
    headers = {
        ‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.7.5000‘}

    for i in range(2):
        #第一次请求，获得地区总门店数，第二次请求，爬取所有门店数据持久化到json文件中
        response = get_response(keyword, num)
        if i == 0:
            dict_data = json.loads(response.text)
            num = str(dict_data[‘Table‘][0][‘rowcount‘])
        else:
            fp = open(‘kfc.json‘, ‘w‘, encoding=‘utf-8‘)
            dict_data = json.loads(response.text)
            json.dump(dict_data, fp=fp, ensure_ascii=False)