在golang磁盘写入性能

Question

在下面的代码中，我使用golang中的bufio将消息写入文件。我的磁盘I / O速度约为1000M / s。奇怪的是，当写入文件的大小小于20G时，写入速度约为每秒800M~900M，略低于I / O速度。但是，当文件大小超过21G时，我发现写入速度大约是每秒200M，远低于I / O速度。我不知道为什么，任何人都可以帮助我？谢谢。

package main

import "fmt"
import (
    "os"
    "time"
    "flag"
    "bufio"
)

func main() {
    var pRound = flag.Int64("round", 3500000, "loop round")
    flag.Parse()

    var message string
    for i := 0; i < 1024; i++ {
        message += "1234567890"
    }
    message += "
"

    f, err := os.OpenFile("server", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
    if err != nil {
        fmt.Println(err)
        return
    }
    w := bufio.NewWriter(f)

    var i int64 = 0
    round := *pRound
    start := time.Now()
    for i = 0; i < round; i++ {
        w.WriteString(message)
    }
    w.Flush()
    f.Close()
    end := time.Now()

    nanoseconds := end.Sub(start).Nanoseconds()
    speed := 1000000000 * round / nanoseconds

    fmt.Printf("round: %v
", round)
    fmt.Printf("Nanoseconds: %v
", nanoseconds)
    fmt.Printf("speed: %v
", speed)
}

我想我问过一个愚蠢的问题。关于Vorsprung的回应，我在这里展示我的C代码。我用C语言再次测试它。我用go语言找到相同的结果。我的测试结果是：

./a.out 10000
seconds: 7
milliseconds: 260910
total: 10485760000 Bytes
total: 10000 M
total: 9.76562 G
speed: 1377.24 M/s


./a.out 20000
seconds: 24
milliseconds: 7249
total: 20971520000 Bytes
total: 20000 M
total: 19.5312 G
speed: 833.082 M/s


./a.out 30000
seconds: 80
milliseconds: 518970
total: 31457280000 Bytes
total: 30000 M
total: 29.2969 G
speed: 372.583 M/s



./a.out 40000
seconds: 134
milliseconds: 615910
total: 41943040000 Bytes
total: 40000 M
total: 39.0625 G
speed: 297.142 M/s

以下是我的C代码：

#include <fcntl.h>
#include <errno.h>
#include <limits.h>
#include <sys/uio.h>
#include <time.h>
#include <sys/time.h>
#include <iostream>
#include <string.h>
#include <stdlib.h>

int main(int argc, char* argv[]) {
  if (argc != 2) {
    std::cout << "usage: " << argv[0] << " round" << std::endl;
    return -1;
  }
  int round = atoi(argv[1]);

  int fd = open("file.data", O_CREAT | O_APPEND | O_RDWR, 0666);
  if (fd == -1) {
    std::cout << "open file error: " << strerror(errno) << std::endl;
    return -1;
  }

  struct iovec vec[IOV_MAX];
  int len = 1024;
  for (int i = 0; i < IOV_MAX; i++) {
    vec[i].iov_base = new char[len];
    vec[i].iov_len = len;

    char *buf = NULL;
    for (int j = 0; j < len - 1; j++) {
      buf = (char*)vec[i].iov_base;
      buf[j] = j % 10 + '1';
    }
    buf[len - 1] = '
';
  }

  timeval tv1;
  gettimeofday(&tv1, NULL);
  for (int i = 0; i < round; i++) {
    writev(fd, vec, IOV_MAX);
  }

  close(fd);

  timeval tv2;
  gettimeofday(&tv2, NULL);

  for (int i = 0; i < IOV_MAX; i++) {
     char* buf = (char*)vec[i].iov_base;
     delete[] buf;
  }

  std::cout << "seconds: " << tv2.tv_sec - tv1.tv_sec << std::endl;
  std::cout << "milliseconds: " << tv2.tv_usec - tv1.tv_usec << std::endl;
  int64_t total = int64_t(len) * IOV_MAX * round;
  float t = (tv2.tv_sec - tv1.tv_sec) * 1000000.0 + (tv2.tv_usec - tv1.tv_usec);
  float speed = 1000000.0 * total / t / 1024 / 1024;
  std::cout << "total: " << total << " Bytes" << std::endl;
  std::cout << "total: " << total / 1024.0 / 1024.0 << " M" << std::endl;
  std::cout << "total: " << total / 1024.0 / 1024.0 / 1024.0 << " G" << std::endl;
  std::cout << "speed: " << speed << " M/s" << std::endl;
  return 0;
}

现在我的diskio.go测试结果显示在这里。因为我不知道如何在评论中评论易读的结果，我在这里展示。

time ./diskio -size=4
written: 4294967296B 26237051975ns 4.29GB 26.24s 163.70MB/s
real    0m26.980s
user    0m0.397s
sys     0m4.874s


time ./diskio -size=8
written: 8589934592B 57803019028ns 8.59GB 57.80s 148.61MB/s
real    0m59.192s
user    0m0.813s
sys     0m9.607s



time ./diskio -size=10
written: 10737418240B 68331989999ns 10.74GB 68.33s 157.14MB/s
real    1m10.288s
user    0m0.946s
sys     0m12.024s




time ./diskio -size=20
written: 21474836480B 141169506440ns 21.47GB 141.17s 152.12MB/s
real    2m25.037s
user    0m1.881s
sys     0m24.029s


time ./diskio -size=30
written: 32212254720B 203807569664ns 32.21GB 203.81s 158.05MB/s
real    3m29.345s
user    0m2.925s
sys     0m33.528s

diskio.go来自https://stackoverflow.com/a/47889346/5616809

我想我得到了答案，测试结果是由磁盘缓冲区原因引起的。我用hdparm推荐测试了我的磁盘速度，我得到了这个：

hdparm -Tt /dev/sde1
/dev/sde1:
Timing cached reads:   18166 MB in  2.00 seconds = 9093.93 MB/sec
Timing buffered disk reads: 584 MB in  3.01 seconds = 194.18 MB/sec

因此，当文件大小小于约18166M时，我的程序可能正在将字节写入缓冲区。之后程序正在写入磁盘，因此速度较慢。

Answer 1

另一答案