python core-spark-filtering-data.py

Posted 2021-05-09

#Filtering the Data
orders = sc.textFile("/public/retail_db/orders")

ordersStatuses = orders.map(lambda order: order.split(",")[3])
for orderStatus in orderStatuses.collect(): print(orderStatus)

ordersFiltered = orders.\
filter(lambda order: order.split(",")[3] == "COMPLETE" or order.split(",")[3] == "CLOSED")

for order in ordersFiltered.take(10): print(order)