python pyspark-wordcount-mapPartitions.py

Posted 2021-05-09

path = "/Users/itversity/Research/data/wordcount.txt" or path = "/public/randomtextwriter/part-m-00000"

def getTuples(lines):
    tuples = [ ]
    for line in lines:
        for i in line.split(" "):
            tuples.append((i, 1))
    return tuples

for i in sc.textFile(path). \
    mapPartitions(lambda lines: getTuples(lines)). \
    reduceByKey(lambda t, e: t + e). \
    take(100):
    print(i)