Spark:WordCount 步骤

Posted 2022-11-28 Frank201608

scala> val data = sc.textFile("/library/wordcount/input/Data")
//data: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[9]  at textFile as <console>:27

scala> val flatted = data.flatMap(_.split(" "))
//flatted: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[10] as flatMap as <console>:29

scala> flatted .toDebugString
res5: String = 
(88) MapPartitionsRDD[10] as flatMap as <console>:29
|    MapPartitionsRDD[9]  at textFile as <console>:27
|    /library/wordcount/input/Data HadoopRDD[8] at textFile as <console>:27


scala> val mapped = flatted .map(word ==>(word,1))
//mapped : org.apache.spark.rdd.RDD[(String, Int)] =  MapPartitionsRDD[11] at map as <console>:31


scala> mapped.toDebugString
(88) MapPartitionsRDD[11] at map      at <console>:31 []
|    MapPartitionsRDD[10] at flatMap  at <console>:29 []
|    MapPartitionsRDD[9]  at textFile at <console>:27 []
|    /library/wordcount/input/Data HadoopRDD[8] at textFile at <console>:27 []

scala> val reduced = mapped.reduceByKey(_+_)
reduced: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[12] as  reduceByKey as <console>:33

scala> reduced.toDebugString
(88)   ShuffledRDD[12]      at reduceByKey as  <console>:33 []
+-(88) MapPartitionsRDD[11] at map at <console>:31 []
|      MapPartitionsRDD[10] at flatMap as <console>:29 []
|      MapPartitionsRDD[9]  at textFile as <console>:27 []
|      /library/wordcount/input/Data HadoopRDD[8] at textFile at <console>:27 []

scala> reduced.savaAsTextFile("/library/wordcount/output")

DAG图如下：