5.2 nc + JMX查看分布式程序数据

Posted 2021-01-12 lybpy

JMX即java管理扩展服务

1、先在s101开启nc服务端

nc -lk 8888

2、先开启spark shell

spark-shell --master spark:s101:7077

3、使用shell

//工具函数，将分布式程序中所需数据回传到执行节点
def sendInfo(obj:Object ,m:String , param:String)= {
    val ip = java.net.InetAddress.getLocalHost.getHostAddress
    val pid = java.lang.management.ManagementFactory.getRuntimeMXBean.getName.split("@")(0)
    val tname = Thread.currentThread().getName
    val classname = obj.getClass.getSimpleName
    val objHash = obj.hashCode()
    val info = "ip:" + ip + "/" + "pid:" + pid + "/" +"当前线程:" +  tname + "/" + "当前对象:" + 
               classname + "@" + objHash + "/" +"方法:" +  m + "/" + "文本:" + param + "
"

    //发送数据给nc 服务器
    val sock = new java.net.Socket("s101" , 8888)
    val out = sock.getOutputStream
    out.write(info.getBytes())
    out.flush()
    out.close()
}

val rdd1 = sc.textFile("myspark/wc" , 3)
val rdd2 = rdd1.flatMap(line=>{
sendInfo(this , "flatMap" , line)
line.split(" ")}
)
val rdd3 = rdd2.coalesce(4,true)
val rdd4 = rdd3.map(word=>{
sendInfo(this , "map" , word)
(word,1)
})
val rdd5 = rdd4.reduceByKey((a,b)=>{
sendInfo(this, "reduceByKey", a + "," + b)
a + b
},5)
val arr = rdd5.collect()

数据

ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]2033283521/方法:flatMap/文本:8 6 7
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]2033283521/方法:flatMap/文本:9 8 9
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]2033283521/方法:flatMap/文本:0 5 6

ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:7
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:0
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:7
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:3
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:3
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]1991892807/方法:map/文本:8

ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]157045983/方法:reduceByKey/文本:1,1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]157045983/方法:reduceByKey/文本:1,1

ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]578058632/方法:reduceByKey/文本:1,1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]578058632/方法:reduceByKey/文本:2,1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-1/当前对象:[email protected]578058632/方法:reduceByKey/文本:2,1

ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-0/当前对象:[email protected]437061105/方法:reduceByKey/文本:1,1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-0/当前对象:[email protected]437061105/方法:reduceByKey/文本:1,1
ip:192.168.137.102/pid:7070/当前线程:Executor task launch worker-0/当前对象:[email protected]437061105/方法:reduceByKey/文本:2,2
------------------------------------------------------------------------------------------------------------------------------------------
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]774135418/方法:flatMap/文本:1 2 3
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]774135418/方法:flatMap/文本:2 4 5
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]774135418/方法:flatMap/文本:3 6 7
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]774135418/方法:flatMap/文本:4 8 9

ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:0
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:3
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:2
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:5
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:4
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:6
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1721291521/方法:map/文本:9

ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:1
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:4
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:7
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:5
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:2
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:5
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:8
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:8
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1352742443/方法:map/文本:6

ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1645250114/方法:reduceByKey/文本:1,1
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1645250114/方法:reduceByKey/文本:1,1

ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1398861210/方法:reduceByKey/文本:1,1
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1398861210/方法:reduceByKey/文本:2,2
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-1/当前对象:[email protected]1398861210/方法:reduceByKey/文本:1,1

ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1022819022/方法:reduceByKey/文本:1,2
ip:192.168.137.103/pid:5996/当前线程:Executor task launch worker-0/当前对象:[email protected]1022819022/方法:reduceByKey/文本:2,1
---------------------------------------------------------------------------------------------------------------------------------------
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]388488515/方法:flatMap/文本:5 0 1
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]388488515/方法:flatMap/文本:6 2 3
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]388488515/方法:flatMap/文本:7 4 5

ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:6
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:6
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:9
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:9
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:4
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:5
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]952442015/方法:map/文本:2

ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]2087943646/方法:reduceByKey/文本:1,1
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]2087943646/方法:reduceByKey/文本:1,1

ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]1851342893/方法:reduceByKey/文本:1,1
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]1851342893/方法:reduceByKey/文本:2,1
ip:192.168.137.104/pid:5673/当前线程:Executor task launch worker-0/当前对象:[email protected]1851342893/方法:reduceByKey/文本:2,1