sparkcore sparksql单词统计—切分-聚合

Posted 2021-02-20 八旗格格的家

 1 package com.bawei.core
 2 
 3 import org.apache.spark.rdd.RDD
 4 import org.apache.spark.{SparkConf, SparkContext}
 5 
 6 /**
 7  * 单词统计
 8 
 9  */
10 object SparkWC {
11   def main(args: Array[String]): Unit = {
12     var conf = new SparkConf().setAppName("SparkWC").setMaster("local[3]")
13     //获取SparkContext对象
14     var sc = new SparkContext(conf)
15 
16     //1、读取文件生成数据RDD
17     var dataRdd = sc.textFile("a.txt")
18     //切分
19     var flatRdd = dataRdd.flatMap(_.split(" "))
20     //标记
21     //var mapRdd = flatRdd.map((_,1))
22     val mapRdd: RDD[(String, Int)] = flatRdd.map((_, 1))
23     //聚合
24     var reduceRdd = mapRdd.reduceByKey(_ + _)
25     //打印输出
26     reduceRdd.saveAsTextFile("./result")
27 
28   }
29 }