ceph 简单测试

Posted 2020-10-31

ceph 为3台，3mon 3mgr 9osd，网卡千兆交换机千兆，只进行了内核最大文件数和进程数调整，4核cpu 16G内存
希捷硬盘为1t

#利用for循环多次测试
for i in {1..10};do fio xxx >>suijixie.txt ;done
egrep "^  write" /home/data/suijixie.txt 查看结果
#测试随机写
fio -direct=1 -iodepth=128 -rw=randwrite -ioengine=libaio -bs=4k -size=1G -numjobs=1 -runtime=1000 -group_reporting -filename=iotest -name=Rand_Write_Testing

egrep "^  write" /home/data/suijidu.txt
#rbd
write: IOPS=497, BW=1989KiB/s (2037kB/s)(1024MiB/527210msec)
write: IOPS=726, BW=2906KiB/s (2976kB/s)(1024MiB/360832msec)
write: IOPS=739, BW=2958KiB/s (3029kB/s)(1024MiB/354431msec)
write: IOPS=748, BW=2993KiB/s (3065kB/s)(1024MiB/350307msec)
write: IOPS=756, BW=3026KiB/s (3098kB/s)(1024MiB/346545msec)
write: IOPS=679, BW=2719KiB/s (2784kB/s)(1024MiB/385679msec)
write: IOPS=676, BW=2705KiB/s (2770kB/s)(1024MiB/387629msec)
write: IOPS=681, BW=2725KiB/s (2790kB/s)(1024MiB/384830msec)
write: IOPS=680, BW=2721KiB/s (2787kB/s)(1024MiB/385300msec)
write: IOPS=678, BW=2716KiB/s (2781kB/s)(1024MiB/386090msec)
write: IOPS=680, BW=2723KiB/s (2788kB/s)(1024MiB/385098msec)
write: IOPS=676, BW=2706KiB/s (2771kB/s)(1024MiB/387547msec)
write: IOPS=680, BW=2722KiB/s (2788kB/s)(1024MiB/385169msec)
write: IOPS=682, BW=2729KiB/s (2794kB/s)(1024MiB/384280msec)
#希捷stat
write: IOPS=640, BW=2564KiB/s (2625kB/s)(1024MiB/409010msec)
write: IOPS=713, BW=2856KiB/s (2924kB/s)(1024MiB/367168msec)
write: IOPS=667, BW=2671KiB/s (2735kB/s)(1024MiB/392526msec)
write: IOPS=713, BW=2852KiB/s (2921kB/s)(1024MiB/367606msec)
write: IOPS=706, BW=2827KiB/s (2894kB/s)(1024MiB/370971msec)
write: IOPS=707, BW=2829KiB/s (2897kB/s)(1024MiB/370659msec)
write: IOPS=704, BW=2820KiB/s (2888kB/s)(1024MiB/371852msec)
write: IOPS=707, BW=2830KiB/s (2898kB/s)(1024MiB/370529msec)
write: IOPS=707, BW=2829KiB/s (2896kB/s)(1024MiB/370708msec)
write: IOPS=707, BW=2830KiB/s (2898kB/s)(1024MiB/370515msec)
write: IOPS=705, BW=2823KiB/s (2890kB/s)(1024MiB/371489msec)
write: IOPS=707, BW=2830KiB/s (2897kB/s)(1024MiB/370577msec)
write: IOPS=706, BW=2825KiB/s (2892kB/s)(1024MiB/371219msec)
#测试随机读
fio -direct=1 -iodepth=128 -rw=randread -ioengine=libaio -bs=4k -size=1G -numjobs=1 -runtime=1000 -group_reporting -filename=iotest -name=Rand_Read_Testing

egrep "^   read" /home/data/suijidu.txt 
#rbd
read: IOPS=569, BW=2278KiB/s (2333kB/s)(1024MiB/460241msec)
read: IOPS=1029, BW=4117KiB/s (4216kB/s)(1024MiB/254699msec)
read: IOPS=1353, BW=5415KiB/s (5545kB/s)(1024MiB/193642msec)
read: IOPS=13.3k, BW=52.0MiB/s (54.6MB/s)(1024MiB/19679msec)
read: IOPS=33.4k, BW=130MiB/s (137MB/s)(1024MiB/7856msec)
read: IOPS=33.3k, BW=130MiB/s (136MB/s)(1024MiB/7869msec)
read: IOPS=33.6k, BW=131MiB/s (138MB/s)(1024MiB/7809msec)
   read: IOPS=1399, BW=5596KiB/s (5731kB/s)(1024MiB/187370msec)
   read: IOPS=33.5k, BW=131MiB/s (137MB/s)(1024MiB/7814msec)
   read: IOPS=32.8k, BW=128MiB/s (135MB/s)(1024MiB/7981msec)
   read: IOPS=33.2k, BW=130MiB/s (136MB/s)(1024MiB/7904msec)
   read: IOPS=33.5k, BW=131MiB/s (137MB/s)(1024MiB/7818msec)
   read: IOPS=33.4k, BW=130MiB/s (137MB/s)(1024MiB/7860msec)
   read: IOPS=33.4k, BW=130MiB/s (137MB/s)(1024MiB/7848msec)
   read: IOPS=33.3k, BW=130MiB/s (136MB/s)(1024MiB/7877msec)
   read: IOPS=33.2k, BW=130MiB/s (136MB/s)(1024MiB/7884msec)
   read: IOPS=33.4k, BW=131MiB/s (137MB/s)(1024MiB/7838msec)
#希捷stat
read: IOPS=667, BW=2672KiB/s (2736kB/s)(1024MiB/392479msec)
read: IOPS=668, BW=2674KiB/s (2739kB/s)(1024MiB/392067msec)
read: IOPS=666, BW=2668KiB/s (2732kB/s)(1024MiB/393056msec)
read: IOPS=665, BW=2661KiB/s (2725kB/s)(1024MiB/394044msec)
read: IOPS=667, BW=2669KiB/s (2733kB/s)(1024MiB/392855msec)
read: IOPS=666, BW=2665KiB/s (2728kB/s)(1024MiB/393530msec)
read: IOPS=664, BW=2659KiB/s (2723kB/s)(1024MiB/394358msec)
read: IOPS=665, BW=2661KiB/s (2725kB/s)(1024MiB/394011msec)
read: IOPS=640, BW=2563KiB/s (2624kB/s)(930MiB/371585msec)
read: IOPS=666, BW=2665KiB/s (2728kB/s)(1024MiB/393540msec)

#测试顺序写吞吐量
fio -direct=1 -iodepth=64 -rw=write -ioengine=libaio -bs=1024k -size=1G -numjobs=1 -runtime=1000 -group_reporting -filename=iotest -name=Write_PPS_Testing

egrep "^  write" /home/data/dutuxie.txt 
#rbd
write: IOPS=108, BW=109MiB/s (114MB/s)(1024MiB/9398msec)
write: IOPS=109, BW=110MiB/s (115MB/s)(1024MiB/9347msec)
write: IOPS=109, BW=109MiB/s (114MB/s)(1024MiB/9388msec)
write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9368msec)
  write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9368msec)
  write: IOPS=109, BW=109MiB/s (114MB/s)(1024MiB/9381msec)
  write: IOPS=109, BW=109MiB/s (114MB/s)(1024MiB/9384msec)
  write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9363msec)
  write: IOPS=109, BW=109MiB/s (114MB/s)(1024MiB/9392msec)
  write: IOPS=109, BW=110MiB/s (115MB/s)(1024MiB/9346msec)
  write: IOPS=109, BW=109MiB/s (114MB/s)(1024MiB/9389msec)
  write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9373msec)
  write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9361msec)
  write: IOPS=109, BW=109MiB/s (115MB/s)(1024MiB/9352msec)
#希捷stat
write: IOPS=171, BW=172MiB/s (180MB/s)(1024MiB/5954msec)
 write: IOPS=216, BW=217MiB/s (227MB/s)(1024MiB/4729msec)
write: IOPS=215, BW=216MiB/s (226MB/s)(1024MiB/4741msec)
write: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4740msec)
write: IOPS=216, BW=217MiB/s (227MB/s)(1024MiB/4724msec)
write: IOPS=217, BW=217MiB/s (228MB/s)(1024MiB/4717msec)
write: IOPS=215, BW=215MiB/s (226MB/s)(1024MiB/4757msec)
write: IOPS=213, BW=213MiB/s (224MB/s)(1024MiB/4798msec)
write: IOPS=215, BW=215MiB/s (226MB/s)(1024MiB/4756msec)
write: IOPS=214, BW=215MiB/s (225MB/s)(1024MiB/4768msec)
write: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4739msec)
write: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4735msec)
测试顺序读吞吐量
fio -direct=1 -iodepth=64 -rw=read -ioengine=libaio -bs=1024k -size=1G -numjobs=1 -runtime=1000 -group_reporting -filename=iotest -name=Read_PPS_Testing

egrep "^   read" /home/data/tutudu.txt
#rbd
read: IOPS=162, BW=163MiB/s (171MB/s)(1024MiB/6292msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6281msec)
read: IOPS=162, BW=163MiB/s (171MB/s)(1024MiB/6284msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6276msec)
read: IOPS=163, BW=164MiB/s (172MB/s)(1024MiB/6257msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6264msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6264msec)
read: IOPS=163, BW=164MiB/s (172MB/s)(1024MiB/6256msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6282msec)
read: IOPS=163, BW=164MiB/s (172MB/s)(1024MiB/6253msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6264msec)
read: IOPS=163, BW=163MiB/s (171MB/s)(1024MiB/6272msec)
read: IOPS=163, BW=164MiB/s (172MB/s)(1024MiB/6259msec)
#希捷stat
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4731msec)
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4731msec)
read: IOPS=216, BW=217MiB/s (227MB/s)(1024MiB/4726msec)
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4733msec)
read: IOPS=214, BW=215MiB/s (225MB/s)(1024MiB/4771msec)
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4734msec)
read: IOPS=215, BW=216MiB/s (226MB/s)(1024MiB/4743msec)
read: IOPS=215, BW=216MiB/s (226MB/s)(1024MiB/4746msec)
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4735msec)
read: IOPS=215, BW=216MiB/s (226MB/s)(1024MiB/4743msec)
read: IOPS=214, BW=214MiB/s (225MB/s)(1024MiB/4774msec)
read: IOPS=216, BW=216MiB/s (227MB/s)(1024MiB/4740msec)