python pandas使用一些协程

Posted 2021-01-31 hailuo

import pandas as pd

def coroutine(func):
	"""装饰器:向前执行到第一个`yield`表达式,预激`func`"""
	@wraps(func)
	def primer(*args,**kwargs):
		gen = func(*args,**kwargs)
		next(gen)
		return gen
	primer.__name__ = func.__name__
	primer.__dict__ = func.__dict__
	primer.__doc__  = func.__doc__
	return primer

@coroutine
def getd():
	grouped=pd.DataFrame()
	while True:
		chunk=yield grouped
		if chunk is None:
			break
		chunk[‘cishu‘]=1
		chunkgrouped=chunk.groupby(‘somekey‘,as_index=False).sum()
		newchunk=pd.concat([grouped,chunkgrouped],ignore_index=True)
		grouped=newchunk.groupby(‘somekey‘, as_index=False).sum()
	return grouped

cor=getd()
chunks=pd.read_csv(path,low_memory=False,dtype=‘object‘,chunksize=10)
for chunk in chunks:
	cor.send(chunk)
try:
	cor.send(None)
except StopIteration as exc:
	result = exc.value
	print(result)