Spark读取Mysql，Redis，Hbase数据

Posted 2020-11-28 知晓的老巢

1、读取mysql数据

object JdbcRddDemo {

  def getConn() = {
    Class.forName("com.mysql.jdbc.Driver").newInstance()
    DriverManager.getConnection("jdbc:mysql://hadoop000:3306/hive", "root", "root")
  }

  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("jdbcRdd").setMaster("local[*]")
    val sc = new SparkContext(sparkConf)

    val jdbcRDD = new JdbcRDD(
      sc,
      getConn,
      "select * from TBLS where TBL_ID >= ? and TBL_ID <= ?",
      1,
      10,
      2,
      rs => {
        val id = rs.getInt(1)
        val name = rs.getString(2)
        val age = rs.getInt(3)
        (id,name,age)
      }
    )

    jdbcRDD.collect().toBuffer
  }
}