sspark自定义排序

Posted 2020-09-05 willian

/**
  * Created by willian on 2017/3/19.
  * 自定义排序，例如 年龄相同 再比较颜值
  */
object CustomSort {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("flow_analysis").setMaster("local")
    val sc = new SparkContext(conf)
    val person_rdd: RDD[(String, Int, Int)] = sc.parallelize(List(("zhangweilun",20,18),("lixueping",20,19)))
    val sorted_rdd: RDD[(String, Int, Int)] = person_rdd.sortBy(item =>{
      Person(item._3,item._2,item._1)
    },ascending = false)
    println(sorted_rdd.collect().toBuffer)
  }
}

//注意：必须实现Serializable接口，并且集成orderd，重写比较方法
case class Person(var look:Int,var age:Int,var name:String) extends Ordered[Person] with Serializable{
  override def compare(that: Person): Int = {
    if (this.look == that.look){
      that.age - that.age
    }else{
      this.look - that.look
    }
  }
}

如上，加入存储数据的类，并重写比较方法即可