golang ioutil 包源码阅读

Posted 2020-12-10 leafs99

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了golang ioutil 包源码阅读相关的知识，希望对你有一定的参考价值。

func ReadAll(r io.Reader) ([]byte, error)
func ReadFile(filename string) ([]byte, error)
func WriteFile(filename string, data []byte, perm os.FileMode) error
func ReadDir(dirname string) ([]os.FileInfo, error)
func NopCloser(r io.Reader) io.ReadCloser
var Discard io.Writer
func TempFile(dir, pattern string) (f *os.File, err error)
func TempDir(dir, prefix string) (name string, err error)

ioutil 包实现了一些常用的 I/O 函数。

func ReadAll(r io.Reader) ([]byte, error)

ReadAll 读取 r 中所有数据，EOF 不会作为 error 返回。

// readAll reads from r until an error or EOF and returns the data it read
// from the internal buffer allocated with a specified capacity.
// readAll 从 r 中读取所有数据，直到遇到 error 或者 EOF，不会返回 EOF 错误
func readAll(r io.Reader, capacity int64) (b []byte, err error) {
	var buf bytes.Buffer
	// If the buffer overflows, we will get bytes.ErrTooLarge.
	// Return that as an error. Any other panic remains.
	defer func() {
		e := recover()
		if e == nil {
			return
		}
		if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge {
			err = panicErr
		} else {
			panic(e)
		}
	}()
	if int64(int(capacity)) == capacity {
		buf.Grow(int(capacity))
	}
  // buf.ReadFrom(r) 不会返回 EOF 错误
	_, err = buf.ReadFrom(r)
	return buf.Bytes(), err
}

// ReadAll reads from r until an error or EOF and returns the data it read.
// A successful call returns err == nil, not err == EOF. Because ReadAll is
// defined to read from src until EOF, it does not treat an EOF from Read
// as an error to be reported.
func ReadAll(r io.Reader) ([]byte, error) {
	return readAll(r, bytes.MinRead)
}

func ReadFile(filename string) ([]byte, error)

ReadFile 读取文件所有数据（如果文件不是太大的话），返回读到的数据和 error，不会返回 EOF 错误

// ReadFile reads the file named by filename and returns the contents.
// A successful call returns err == nil, not err == EOF. Because ReadFile
// reads the whole file, it does not treat an EOF from Read as an error
// to be reported.
func ReadFile(filename string) ([]byte, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It‘s a good but not certain bet that FileInfo will tell us exactly how much to
	// read, so let‘s try it but be prepared for the answer to be wrong.
	var n int64 = bytes.MinRead

	if fi, err := f.Stat(); err == nil {
		// As initial capacity for readAll, use Size + a little extra in case Size
		// is zero, and to avoid another allocation after Read has filled the
		// buffer. The readAll call will read into its allocated internal buffer
		// cheaply. If the size was wrong, we‘ll either waste some space off the end
		// or reallocate as needed, but in the overwhelmingly common case we‘ll get
		// it just right.
    // 作为 readAll 的初始容量，在大小为零的情况下使用 Size + 一些额外的空间，
    // 并在 Read 填充缓冲区后避免再次分配。 readAll 调用将廉价地读取其分配的内部缓冲区。
    // 如果大小错误，我们要么浪费一些空间，要么根据需要重新分配，
    // 但是在绝大多数情况下，我们会使其正确。
		if size := fi.Size() + bytes.MinRead; size > n {
			n = size
		}
	}
	return readAll(f, n)
}

func WriteFile(filename string, data []byte, perm os.FileMode) error

WriteFile 将数据写进 fileName 的文件里，如果文件不存在，就以 perm 的权限创建文件，否则就先清空文件再写数据

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm os.FileMode) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}

func ReadDir(dirname string) ([]os.FileInfo, error)

ReadDir 读取 dirname 中的所有子目录或者文件，结果按照文件名排序

// ReadDir reads the directory named by dirname and returns
// a list of directory entries sorted by filename.
func ReadDir(dirname string) ([]os.FileInfo, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
	return list, nil
}

func NopCloser(r io.Reader) io.ReadCloser

NopCloser 输入一个 Reader，返回一个 ReadCloser，其中的 Close 方法不会做任何操作

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() error { return nil }

// NopCloser returns a ReadCloser with a no-op Close method wrapping
// the provided Reader r.
func NopCloser(r io.Reader) io.ReadCloser {
	return nopCloser{r}
}

用法：

// server	
body, err := ioutil.ReadAll(ctx.Request.Body)
if err != nil {
  return err
}
ctx.Request.Body = ioutil.NopCloser(bytes.NewReader(body))

Client 端生成的 request body 不用 close，因为 net/http 中的 func (c *Client) Do(req *Request) (*Response, error) 会调用 Close()，但是接收到的 response 需要关闭，否则会造成 GC 回收不到，产生资源泄露。

server 端接收到的 request body 也是需要关闭的，由 server 自动关闭，The Server will close the request body. The ServeHTTP Handler does not need to.

var Discard io.Writer

Discard 实现了 io.Writer，它的所有写操作都会成功，但是却不会做任何事，还实现了 io.ReadFrom，所以 io.Copy 会直接读出数据然后丢弃。

type devNull int

// devNull implements ReaderFrom as an optimization so io.Copy to
// ioutil.Discard can avoid doing unnecessary work.
var _ io.ReaderFrom = devNull(0)

func (devNull) Write(p []byte) (int, error) {
	return len(p), nil
}

// 这个方法有什么用呢？？？？？？？？？？？？？？？
func (devNull) WriteString(s string) (int, error) {
	return len(s), nil
}

var blackHolePool = sync.Pool{
	New: func() interface{} {
		b := make([]byte, 8192)
		return &b
	},
}

func (devNull) ReadFrom(r io.Reader) (n int64, err error) {
	bufp := blackHolePool.Get().(*[]byte)
	readSize := 0
	for {
		readSize, err = r.Read(*bufp)
		n += int64(readSize)
		if err != nil {
			blackHolePool.Put(bufp)
			if err == io.EOF {
				return n, nil
			}
			return
		}
	}
}

// Discard is an io.Writer on which all Write calls succeed
// without doing anything.
var Discard io.Writer = devNull(0)

用法：

a := strings.NewReader("hello")
p := make([]byte, 20)
io.Copy(ioutil.Discard, a)
ioutil.Discard.Write(p)
fmt.Println(p)
// [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

devNull 实现了 ReadFrom，因此 io.Copy 到 ioutil.Discard 避免了不必要的工作，因此其一定会成功．但是 ioutil.Discard 不记录 copy 得到的数值。

func TempFile(dir, pattern string) (f *os.File, err error)

TempFile 创建一个临时文件，输入的 pattern 为 "prefix*suffix" 的形式，该方法会将 "*" 替换成随机数字串作为文件名，返回创建的文件对象和程序运行中产生的错误。

如果 dir 为空，则会在系统的临时目录中创建临时文件，如果环境变量中没有设置系统临时目录，那么则会在 /tmp 目录下创建临时文件，调用者可以通过 f.Name() 方法获得文件的完整路径

注：调用者创建的临时文件应该由创建者自己删除


// Random number state.
// We generate random temporary file names so that there‘s a good
// chance the file doesn‘t exist yet - keeps the number of tries in
// TempFile to a minimum.
var rand uint32
var randmu sync.Mutex

func reseed() uint32 {
	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}

func nextRandom() string {
	randmu.Lock()
	r := rand
	if r == 0 {
		r = reseed()
	}
	r = r*1664525 + 1013904223 // constants from Numerical Recipes
	rand = r
	randmu.Unlock()
	return strconv.Itoa(int(1e9 + r%1e9))[1:]
}

// TempFile creates a new temporary file in the directory dir,
// opens the file for reading and writing, and returns the resulting *os.File.
// The filename is generated by taking pattern and adding a random
// string to the end. If pattern includes a "*", the random string
// replaces the last "*".
// If dir is the empty string, TempFile uses the default directory
// for temporary files (see os.TempDir).
// Multiple programs calling TempFile simultaneously
// will not choose the same file. The caller can use f.Name()
// to find the pathname of the file. It is the caller‘s responsibility
// to remove the file when no longer needed.
func TempFile(dir, pattern string) (f *os.File, err error) {
	if dir == "" {
		dir = os.TempDir()
	}

	var prefix, suffix string
	if pos := strings.LastIndex(pattern, "*"); pos != -1 {
		prefix, suffix = pattern[:pos], pattern[pos+1:]
	} else {
		prefix = pattern
	}

	nconflict := 0
	for i := 0; i < 10000; i++ {
		name := filepath.Join(dir, prefix+nextRandom()+suffix)
		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
		if os.IsExist(err) {
			if nconflict++; nconflict > 10 {
				randmu.Lock()
				rand = reseed()
				randmu.Unlock()
			}
			continue
		}
		break
	}
	return
}

func TempDir(dir, prefix string) (name string, err error)

TempDir 创建一个临时目录，其中 prefix 是文件夹名称的前缀，最终的文件夹名称格式为 prefix+随机字符串，同时调用 TempDir 的多个程序不会选择同一目录

注：调用者创建的临时目录应该由创建者自己删除

// TempDir creates a new temporary directory in the directory dir
// with a name beginning with prefix and returns the path of the
// new directory. If dir is the empty string, TempDir uses the
// default directory for temporary files (see os.TempDir).
// Multiple programs calling TempDir simultaneously
// will not choose the same directory. It is the caller‘s responsibility
// to remove the directory when no longer needed.
func TempDir(dir, prefix string) (name string, err error) {
	if dir == "" {
		dir = os.TempDir()
	}

	nconflict := 0
	for i := 0; i < 10000; i++ {
		try := filepath.Join(dir, prefix+nextRandom())
		err = os.Mkdir(try, 0700)
		if os.IsExist(err) {
			if nconflict++; nconflict > 10 {
				randmu.Lock()
				rand = reseed()
				randmu.Unlock()
			}
			continue
		}
		if os.IsNotExist(err) {
			if _, err := os.Stat(dir); os.IsNotExist(err) {
				return "", err
			}
		}
		if err == nil {
			name = try
		}
		break
	}
	return
}

以上是关于golang ioutil 包源码阅读的主要内容，如果未能解决你的问题，请参考以下文章

Go语言自学系列 | golang标准库ioutil包

golang 文件操作

golang The system cannot find the file specified

Go语言ioutil包详解

Golang读写文件的几种方式

Golang文件IO 一