python异步爬取数据并存储

Posted 2021-07-15 _刘文凯_

如何爬取http网页，并储存在本地

代码如下：

import asyncio
from aiohttp import ClientSession
import aiofiles
import pandas as pd
import os

### 建立本地储存数据文件夹
root_file = './save_file'
if not os.path.exists(root_file):
    os.mkdir('./save')
root_file = root_file + '/'


async def get_url(session, url_value):
    url = "https://www.baidu.com/" + str(url_value) + ".htlm"  # 重点！！！ 在这里构造具体的url
    async with session.get(url) as response:
        status = response.status
        if status == 200:
            response = await response.text() # 读取网页信息
            filename = root_file + url_value + '.htlm'
            async with aiofiles.open(filename, 'w', encoding='utf-8') as fp:
                await fp.write(response)   # 保存为htlm
        else: # 如果打开失败，则打印错误信息
            print('错误：{}, code: {}'.format(url_value,status))
    return [url, status]


async def main(all_singlevalue):

    url_values_list = all_singlevalue
    return_status = []
    async with ClientSession() as session:
        for url in url_values_list:
            task_list = [asyncio.create_task(get_url(session, url))]
        done_data, set_yes = await asyncio.wait(task_list)

        for i_re in done_data:
            return_status.append(i_re.result())

        save_pd = pd.DataFrame(return_status, columns=['url_value', 'status']) # 记录状态码
        save_pd.to_csv('status_log.csv')


if __name__ == '__main__':

    all_value = data['value']  # url变化关键字（列表形式）
    all_singlevalue= all_value .drop_duplicates(keep='first') # 去除可能的重复

    loop = asyncio.get_event_loop()
    loop.run_until_complete(main(all_singlevalue))

    print('全部完成...')