从Spark-Shell到SparkContext的函数调用路径过程分析（源码）

Posted 2020-08-10 大数据和人工智能躺过的坑

　　

　  不急，循序渐进，先打好基础

    Spark shell的原理

　　

首先，我们清晰定位找到这几个。

 

1、spark-shell

 

 

2、 spark-submit

 

 

3、spark-class

 

 

 

 4、SparkSubmit.scala

 

 

 

 

 

 5、SparkILoop.scala

 

 

initializeSpark的源码

def initializeSpark() {
  intp.beQuietDuring {
    command("""
       @transient val sc = {
         val _sc = org.apache.spark.repl.Main.interp.createSparkContext()
         println("Spark context available as sc.")
         _sc
       }
      """)
    command("""
       @transient val sqlContext = {
         val _sqlContext = org.apache.spark.repl.Main.interp.createSQLContext()
         println("SQL context available as sqlContext.")
         _sqlContext
       }
      """)
    command("import org.apache.spark.SparkContext._")
    command("import sqlContext.implicits._")
    command("import sqlContext.sql")
    command("import org.apache.spark.sql.functions._")
  }

 

 createSparkContext的源码

// NOTE: Must be public for visibility
@DeveloperApi
def createSparkContext(): SparkContext = {
  val execUri = System.getenv("SPARK_EXECUTOR_URI")
  val jars = SparkILoop.getAddedJars
  val conf = new SparkConf()
    .setMaster(getMaster())
    .setAppName("Spark shell")
    .setJars(jars)
    .set("spark.repl.class.uri", intp.classServerUri)
  if (execUri != null) {
    conf.set("spark.executor.uri", execUri)
  }
  sparkContext = new SparkContext(conf)
  logInfo("Created spark context..")
  sparkContext
}

 

 

 

总结