spark案例——wordcount

Posted 2021-10-29 Z-hhhhh

local本地模式

添加pom依赖

<properties>
    <scala.version>2.12.0</scala.version>
  </properties>

<dependencies>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
      <version>3.0.0</version>
    </dependency>
  </dependencies>

自行准备一个文件word.txt

//创建spark运行配置对象
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    //创建spark上下文对象(连接对象)
    val sc: SparkContext = new SparkContext(sparkConf)

    //读取文件
    val file: RDD[String] = sc.textFile("src/main/inputfile/word.txt")
	//如果文件在hdfs上
	// val file: RDD[String] = sc.textFile("hdfs://ip地址:9820/study/sparktest/word.txt")

    //计算
    file.flatMap(_.split(" "))
      .map((_,1))
      .reduceByKey(_+_)
      .collect()
      .foreach(println)

    sc.stop()

如果想要保存结果

 sc.textFile("src/main/inputfile/word.txt")
      .flatMap(_.split(" "))
      .map((_, 1))
      .reduceByKey(_ + _)
      .repartition(1)
        .saveAsTextFile("地址")