spark常用参数

Posted 2021-02-01 moonlightml

 val conf = new SparkConf().setAppName("WordCount_groupBy").setMaster("local")
      // .set("spark.default.parallelism", "100") //  1. 调节并行度
      .set("spark.executor.memory ","4g")
      .set("spark.executor.cores","5")
      .set("spark.executor.nums","4")//1
      //静态内存机制
      .set("spark.memory.useLegacyMode","false")
      .set("spark.storage.memoryFraction", "0.3")// 5.cache占用的内存占比,default=0.6
      //统一内存机制
      .set("spark.memory.Fraction","0.3")//default=0.6
      .set("spark.storage.storageFraction","0.9")//default=0.5
      .set("spark.shuffle.consolidateFiles", "false")