python Pyspark：双爆炸发生器

Posted 2021-05-09

def dualExplode(row):
    """Explode weights and category_ids list elements to separate rows.
    Args:
        row: Row
    Yield:
        Row(**newDict)
    """
    rowDict = row.asDict()
    xList = rowDict.pop('x')
    yList = rowDict.pop('y')
    for x,y in zip(xList, yList):
        newDict = dict(rowDict)
        newDict['category_ids'] = x
        newDict['weights'] = y
        yield Row(**newDict)
 
# Example usage
exploded_df = sqlContext.createDataFrame(df.rdd.flatMap(dualExplode))