python3-pachong-01-较好的写法

Posted 2022-05-27 Python_Heaven

python3 网络爬虫开发实战 第2版
https://spa1.scrape.center
注意是123的1，
第五章的ajax的分析写法：
觉得作者能够很精炼的用几个传参，将代码简化到3-4个函数就可以爬一个主页，还有详情页，写代码的手法值得学习。哈哈哈

import requests
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s:%(message)s')
INDEX_URL = 'https://spa1.scrape.center/api/movie/?limit=limit&offset=offset'


def scrape_api(url):
    logging.info('scraping %s...', url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.json()
        logging.error('get invalid status code %s while scraping %s', response.status_code, url)
    except requests.RequestException:
        logging.error('error occurred while scraping %s', url, exc_info=True)


LIMIT = 10


def scrape_index(page):
    url = INDEX_URL.format(limit=LIMIT, offset=LIMIT * (page - 1))
    return scrape_api(url)


DETAIL_URL = 'https://spa1.scrape.center/api/movie/id'


def scrape_detail(id):
    url = DETAIL_URL.format(id=id)
    return scrape_api(url)


TOTAL_PAGE = 10


def main():
    for page in range(1, TOTAL_PAGE + 1):
        index_data = scrape_index(page)
        for item in index_data.get("results"):
            id = item.get('id')
            detail_data = scrape_detail(id)
            logging.info('detail data %s', detail_data)


if __name__ == '__main__':
    main()

后面会继续修改本文章内容。