Celery + Redis - .get（）在平稳运行约70个小时后无限期挂起

Question

一切都运行良好多天，但随后我无限期地挂在.get（）上。无限期挂起所需的时间各不相同，但运行时间为24到72小时。

我怀疑这与Redis经纪人有关。 CLIENT LIST中redis-cli的输出显示了大量具有非常高空闲数的连接（见下文）。但我不知道这是否是一个问题，或者为什么这会导致Celery的.get（）无限期挂起。

我已经确认这不是由长时间运行的任务引起的。我还在其中一个工作进程上运行了strace和lsof，但是我无法从输出中收集任何有用的东西：strace output，lsof output。

在我收到通知.get（）挂起后不久，我在主服务器（我有1个主服务器和6个工作服务器）的CLIENT LIST上运行redis-cli。我附加了一部分输出（full output is 659 lines long）。看起来很多都有很高的空闲时间，看起来它列出了我无法识别的IP连接，或者我过去（几周前）用作工作服务器的连接。我目前使用的IP已被[workerIPx]取代。

Redis出了什么问题导致这个.get（）无限期挂起？

addr=67.205.178.82:43310 fd=111 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.82:43311 fd=112 idle=1060095 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.82:43312 fd=113 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.82:43313 fd=114 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.82:43314 fd=115 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=192.241.144.208:43997 fd=119 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=192.241.144.208:43998 fd=120 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=192.241.144.208:43999 fd=121 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=192.241.144.208:44000 fd=122 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=192.241.144.208:44002 fd=23 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=192.241.144.208:44003 fd=24 idle=1060097 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=192.241.144.208:44004 fd=25 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=192.241.144.208:44005 fd=26 idle=1061197 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=192.241.144.208:44006 fd=51 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.82:43317 fd=132 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.82:43318 fd=133 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=192.241.144.208:44009 fd=139 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=192.241.144.208:44010 fd=140 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=192.241.144.208:44011 fd=141 idle=1060097 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.68:34569 fd=148 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.68:34570 fd=149 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.68:34571 fd=150 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.68:34572 fd=151 idle=1060226 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.68:34574 fd=153 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.68:34575 fd=154 idle=1060100 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.68:34576 fd=155 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.68:34577 fd=156 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.68:34578 fd=157 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=192.241.144.208:44013 fd=160 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=192.241.144.208:44015 fd=162 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50152 fd=163 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.188.236:50153 fd=164 idle=1060095 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.188.236:50154 fd=165 idle=1060226 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.188.236:50155 fd=167 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.188.236:50156 fd=168 idle=1060226 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.82:43320 fd=169 idle=1061196 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.82:43321 fd=170 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.82:43322 fd=171 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.82:43323 fd=172 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.68:34580 fd=178 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.68:34581 fd=179 idle=1060100 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.68:34583 fd=181 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.17:50302 fd=183 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.17:50303 fd=184 idle=1060096 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.17:50304 fd=185 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.17:50305 fd=186 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.17:50306 fd=187 idle=1060236 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.68:34586 fd=201 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.68:34587 fd=202 idle=1061195 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50159 fd=205 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50160 fd=206 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.82:43325 fd=208 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.82:43326 fd=209 idle=1060095 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.82:43328 fd=211 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.80:53274 fd=213 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.80:53275 fd=214 idle=1060115 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.80:53276 fd=215 idle=1060227 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.80:53277 fd=216 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.80:53278 fd=217 idle=1061172 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.17:50308 fd=219 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.17:50310 fd=221 idle=1061194 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.80:53280 fd=232 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.80:53282 fd=234 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50162 fd=236 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.188.236:50163 fd=237 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.188.236:50164 fd=238 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.188.236:50165 fd=239 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.80:53284 fd=241 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.80:53285 fd=242 idle=1060114 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.80:53286 fd=243 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.80:53287 fd=244 idle=1061193 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.80:53288 fd=245 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.178.17:50312 fd=247 idle=1061192 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.17:50313 fd=248 idle=1060095 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.17:50314 fd=249 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=67.205.178.17:50315 fd=250 idle=1061192 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.17:50316 fd=251 idle=1061190 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.188.236:50167 fd=253 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50168 fd=255 idle=1060093 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=67.205.178.80:53291 fd=258 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.80:53292 fd=259 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.188.236:50170 fd=260 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.17:50319 fd=282 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.17:50320 fd=283 idle=1061191 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=67.205.178.68:34629 fd=290 idle=1061176 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=192.241.144.208:44052 fd=291 idle=1061176 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.178.68:34633 fd=292 idle=1060216 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=192.241.144.208:44057 fd=294 idle=1060875 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.160.130:35450 fd=588 idle=1051857 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.160.130:35452 fd=589 idle=1051349 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.137.222:33356 fd=590 idle=1051855 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.137.222:33363 fd=591 idle=1051359 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=67.205.137.222:57923 fd=602 idle=1051368 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=67.205.137.222:57925 fd=603 idle=1051349 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=196.52.43.62:6666 fd=665 idle=879946 flags=N db=0 sub=0 psub=0 qbuf=5 obl=0 oll=0 events=r cmd=NULL
addr=[workerIP1]:48367 fd=7 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP1]:48368 fd=8 idle=249155 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP1]:48369 fd=9 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP1]:48370 fd=10 idle=249155 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP1]:48371 fd=11 idle=13082 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP1]:48372 fd=12 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP1]:48373 fd=13 idle=249155 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP1]:48374 fd=14 idle=86 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP1]:48375 fd=15 idle=249155 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP1]:48376 fd=16 idle=1294 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP1]:48377 fd=17 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP1]:48378 fd=18 idle=249154 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP1]:48379 fd=19 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP1]:48380 fd=20 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP1]:48381 fd=21 idle=249154 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP1]:48382 fd=22 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP1]:48383 fd=27 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP1]:48384 fd=28 idle=249153 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP1]:48385 fd=29 idle=249153 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP1]:48386 fd=32 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP2]:45639 fd=34 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP2]:45640 fd=35 idle=249153 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP2]:45641 fd=39 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP2]:45642 fd=40 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP2]:45643 fd=41 idle=249153 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP2]:45644 fd=42 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP5]:56759 fd=43 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP5]:56760 fd=44 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56761 fd=45 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP5]:56762 fd=46 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56763 fd=47 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP3]:53711 fd=48 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP3]:53712 fd=49 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53713 fd=50 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP3]:53714 fd=52 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP3]:53715 fd=53 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53716 fd=54 idle=2846 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP5]:56764 fd=55 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP5]:56765 fd=56 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56766 fd=57 idle=86 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP5]:56767 fd=58 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56768 fd=59 idle=26 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP2]:45645 fd=60 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP2]:45646 fd=61 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP2]:45647 fd=62 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP2]:45648 fd=63 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60568 fd=64 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP4]:60569 fd=65 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP4]:60570 fd=66 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP4]:60571 fd=67 idle=249152 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP4]:60572 fd=68 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP5]:56769 fd=74 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP5]:56770 fd=75 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP5]:56771 fd=76 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP5]:56772 fd=77 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP5]:56773 fd=78 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP2]:45649 fd=79 idle=1 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP2]:45650 fd=80 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP2]:45651 fd=81 idle=85 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP2]:45652 fd=82 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP2]:45653 fd=83 idle=124354 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:40906 fd=84 idle=1 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:40907 fd=86 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:40908 fd=87 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40909 fd=109 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40910 fd=110 idle=53 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[masterIP]:40911 fd=116 idle=92 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[masterIP]:40912 fd=117 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40913 fd=123 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40914 fd=125 idle=20290 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:40915 fd=126 idle=1330 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP6]:57271 fd=127 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP3]:53717 fd=128 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP6]:57272 fd=129 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56774 fd=130 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP5]:56775 fd=131 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP5]:56776 fd=134 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP6]:57273 fd=135 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP3]:53718 fd=136 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP5]:56777 fd=137 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP5]:56778 fd=138 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP3]:53719 fd=142 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP6]:57274 fd=143 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53720 fd=146 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP6]:57275 fd=176 idle=37436 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP4]:60573 fd=190 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP4]:60574 fd=191 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP4]:60575 fd=192 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60576 fd=193 idle=249151 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60577 fd=194 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP2]:45654 fd=195 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP2]:45655 fd=196 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP2]:45656 fd=197 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP2]:45657 fd=207 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60578 fd=225 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP2]:45658 fd=226 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP4]:60579 fd=227 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP4]:60580 fd=228 idle=86 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[workerIP3]:53721 fd=229 idle=0 flags=b db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[workerIP4]:60581 fd=230 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP4]:60582 fd=233 idle=16608 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP3]:53722 fd=240 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53723 fd=252 idle=86 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[masterIP]:40916 fd=254 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[masterIP]:40917 fd=256 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[masterIP]:40918 fd=257 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[masterIP]:40919 fd=261 idle=1 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[masterIP]:40920 fd=262 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[masterIP]:40921 fd=267 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[masterIP]:40922 fd=268 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[masterIP]:40923 fd=281 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[masterIP]:40924 fd=284 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[masterIP]:40925 fd=288 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP3]:53724 fd=297 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53725 fd=298 idle=88 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP6]:57276 fd=299 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP6]:57277 fd=302 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP6]:57278 fd=303 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP6]:57279 fd=304 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP6]:57280 fd=305 idle=249150 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60583 fd=306 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP4]:60584 fd=326 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP4]:60585 fd=333 idle=249149 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60586 fd=335 idle=249149 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP4]:60587 fd=345 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP3]:53726 fd=364 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP3]:53727 fd=371 idle=249149 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[workerIP3]:53728 fd=373 idle=2 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=publish
addr=[workerIP3]:53729 fd=379 idle=1 flags=N db=0 sub=0 psub=1 qbuf=0 obl=0 oll=0 events=r cmd=psubscribe
addr=[workerIP3]:53730 fd=380 idle=249149 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=sadd
addr=[masterIP]:40926 fd=383 idle=249145 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40927 fd=384 idle=249145 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=exec
addr=[masterIP]:40932 fd=387 idle=249140 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40933 fd=391 idle=121 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:40934 fd=392 idle=84 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:40935 fd=393 idle=1892 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=unwatch
addr=[masterIP]:40936 fd=394 idle=249140 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40937 fd=395 idle=63 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:40938 fd=398 idle=249140 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:40940 fd=403 idle=135 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:40941 fd=406 idle=42 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:40942 fd=414 idle=51 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=unwatch
addr=[masterIP]:40946 fd=415 idle=90 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:40947 fd=416 idle=874 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=unwatch
addr=[workerIP5]:56796 fd=417 idle=249139 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56798 fd=418 idle=90846 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP4]:60601 fd=419 idle=249139 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP4]:60603 fd=420 idle=86 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP2]:45688 fd=421 idle=249138 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP2]:45690 fd=422 idle=26 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP3]:53755 fd=423 idle=249138 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP3]:53757 fd=424 idle=26 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:41011 fd=429 idle=249138 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[masterIP]:41012 fd=432 idle=2876 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[masterIP]:41013 fd=434 idle=2395 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=brpop
addr=[masterIP]:41014 fd=437 idle=2396 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=unwatch
addr=[workerIP5]:56880 fd=439 idle=249137 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping
addr=[workerIP5]:56884 fd=441 idle=2844 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=lpush
addr=[workerIP2]:45725 fd=443 idle=249137 flags=N db=0 sub=0 psub=0 qbuf=0 obl=0 oll=0 events=r cmd=ping