spark scala wordcount 例子

Posted 2021-02-04 yehuabin

object ScalaApp {

  def main(args: Array[String]): Unit = {
    var conf = new SparkConf()
    conf.setMaster("local")
    var sc = new SparkContext(conf)
    val lines: RDD[String] = sc.textFile("./words.txt")
    lines.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey((v1,v2)=>v1+v2).foreach(println)
    // 简洁版
    // lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).foreach(println)
  }
}