python pandas使用chunksize异步拆分固定行数的文件

Posted 2021-02-03 hailuo

import pandas as pd
import asyncio
from collections import defaultdict


collect = defaultdict(list)

#######创建处理一个对象的方法，并返回期物
async def dealone(chunk,sas):
	path=‘/data/chaifen/testphone_%d.txt‘%sas
	chunk.to_csv(path,index=False)
	return path

async def submitor(chunk,sas):
	future = await dealone(chunk,sas)
	collect[‘res‘].append(future)
	#print(future)


chunks = pd.read_csv(‘/data/testphone.txt‘,dtype=‘object‘,chunksize=3)

loop = asyncio.get_event_loop()
tasks = asyncio.wait([submitor(chunk,index) for index,chunk in enumerate(chunks,1)])
loop.run_until_complete(tasks)
print(collect[‘res‘])