Spark框架—RDD算式mapPartitionsWithIndex与filter的用法

Posted 2022-03-30 那人独钓寒江雪.

@junit.Test
  def mapPartitionsWithIndex(): Unit =
    sc.parallelize(Seq(1,2,3,4,5,6),2)
      .mapPartitionsWithIndex((index,iter) =>
        println("index:"+index)
        iter.foreach(item=>println(item))
        iter
      )
      .collect()
  
  @junit.Test
  def map1(): Unit =
    sc.parallelize(Seq(1,2,3,4,5,6),2)
      .mapPartitionsWithIndex((index,iter)=>
          println("index:"+index)
        iter.map(item=> item *10)
        iter.foreach(item => println(item))
        iter
      )
      .collect()
  
  @junit.Test
  //1.定义集合
  //2.过滤数据
  //3.收集结果
  def filter(): Unit = //filter相当于if结构
    sc.parallelize(Seq(1,2,3,4,5,6,7,8,9,10))
      .filter(item => item % 2==0)
      .collect()
      .foreach(item => println(item))