Spark-广播变量

Posted 2021-06-30 Mr.zhou_Zxy

Spark-广播变量

广播变量用来高效的分发较大的对象。向所有工作结点发送一个较大的只读值，以供其他的RDD使用。

• 理解记忆

• Code

package com.zxy.spark.Streaming.day005

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object demo3 {
    def main(args: Array[String]): Unit = {
        val sc = new SparkContext(new SparkConf().setAppName("demo3").setMaster("local[*]"))
        //定义广播变量类型
        val mapBroad: Map[String, String] = Map(
            "mi" -> "小米",
            "apple" -> "苹果",
            "lenovo" -> "联想"
        )
        //定义广播变量
        val brandBC: Broadcast[Map[String, String]] = sc.broadcast(mapBroad)
        val ElecRDD: RDD[Electronic_brand] = sc.parallelize(List(
            Electronic_brand(0, "apple"),
            Electronic_brand(1, "mi"),
            Electronic_brand(2, "lenovo")
        ))
        ElecRDD.map(info => {
            val brand: String = brandBC.value.getOrElse(info.EID,"NULL")
            Electronic_brand(info.id,brand)
        }).collect().foreach(println)
    }
}
case class Electronic_brand(id:Int,EID:String)