SparkContext的初始化

Posted 2021-11-10 chengbao

SparkDriver 用于提交用户的应用程序， 

一、SparkConf

负责SparkContext的配置参数加载， 主要通过ConcurrentHashMap来维护各种`spark.*`的配置属性

class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Serializable {

    import SparkConf._

    /** Create a SparkConf that loads defaults from system properties and the classpath */
    def this() = this(true)

    /**
     * 维护一个ConcurrentHashMap 来存储spark配置
     */
    private val settings = new ConcurrentHashMap[String, String]()

    @transient private lazy val reader: ConfigReader = {
        val _reader = new ConfigReader(new SparkConfigProvider(settings))
        _reader.bindEnv(new ConfigProvider {
            override def get(key: String): Option[String] = Option(getenv(key))
        })
        _reader
    }

    if (loadDefaults) {
        loadFromSystemProperties(false)
    }

    /**
     * 加载spark.*的配置
     * @param silent
     * @return
     */
    private[spark] def loadFromSystemProperties(silent: Boolean): SparkConf = {
        // Load any spark.* system properties, 只加载spark.*的配置
        for ((key, value) <- Utils.getSystemProperties if key.startsWith("spark.")) {
            set(key, value, silent)
        }
        this
    }
}

View Code

 

二、SparkContext