Spark广播变量broadcast案例

Posted 2021-06-26 Mr.zhou_Zxy

Spark共享变量之广播变量broadcast

package com.zxy.spark.core.Day04

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD

/**
 * file:broadcast
 * author:zxy
 * date:2021-06-10
 * desc:共享变量之广播变量
 */
object Demo11_share {
    def main(args: Array[String]): Unit = {
        val sc: SparkContext = new SparkContext(new SparkConf().setMaster("local[*]").setAppName("share"))
        //共享变量性别
        val genderMap = Map(
            "0" -> "小姐姐",
            "1" -> "小哥哥"
        )
        //共享变量名字
        val nameMap = Map(
            "1" -> "鸣人",
            "2" -> "博人",
            "3" -> "佐助",
            "4" -> "凯",
            "5" -> "卡卡西"
        )
    
        val listRDD: RDD[(String, String, Int)] = sc.parallelize(List(
            ("1", "1", 30),
            ("2", "0", 18),
            ("3", "1", 31),
            ("4", "0", 40),
            ("5", "2", 41)
        ))
        val genders: Broadcast[Map[String, String]] = sc.broadcast(genderMap)
        val names: Broadcast[Map[String, String]] = sc.broadcast(nameMap)
        listRDD.map(stu => {
            val id: String = stu._1
            val sex: String = stu._2
            val age: Int = stu._3
            
            val gender: String = genders.value.getOrElse(sex,"未知")
            val name: String = names.value.getOrElse(id,"未知")
            Student(id,name,gender,age)
        }).foreach(println)
    }
}
case class Student(id:String,name:String,sex:String,age:Int){}