Java nio 的文件处理

Posted 2020-12-29 Lazy Gene

一、创建一个大文件

下载文件时往往会创建一个指定大小的空文件

package com.lazy.nio;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;

/**
 * 创建大文件  
 * @author Lazy Gene
 *
 */
public class FileCreator {
	
	public static void main(String[] args) {
		FileCreator creator = new FileCreator();
		creator.createBigEmptyFile();
	}
	
	void createBigEmptyFile(){
		
		Path filePath = Paths.get("from/test.tmp");
　　　　　　　　　　// 这段代码实际上可以在FIleChannel 调用open方式时指定OpenOption 为Create_NEW
		try {
			if (!Files.exists(filePath)) {
				Files.createFile(filePath);
			}
		} catch (IOException e1) {
			e1.printStackTrace();
		}
		
		//写一个字节
		ByteBuffer buffer = ByteBuffer.allocate(1);
		try  (FileChannel fileChannel = FileChannel.open(filePath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
			fileChannel.position(2L << 32 - 1); //移动位置, 生成一个4G的空文件
			fileChannel.write(buffer);
		} catch (IOException e) {
			e.printStackTrace();
		}
		
	}
}

　　

 

二、文件转移

NIO 提供transferTo tansferFrom, 和传统的文件访问方式相比减少了数据从内核到用户空间的复制，数据直接在内核移动，在Linux系统中使用sendfile系统调用

这里分别通过FileChannel.transferFrom 和Files.copy以及普通的io调用实现文件的复制

package com.lazy.nio;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;

/**
 * 文件转移
 * 
 * @author Lazy Gene([email protected])
 *
 */
public class FileTransfer {
	Path fromPath = Paths.get("from/test.tmp");
	Path toPath = Paths.get("to/test.tmp");

	public static void main(String[] args) throws IOException {
		FileTransfer transfer = new FileTransfer();
		long start = System.currentTimeMillis();
		transfer.transferFrom();
		long end1 = System.currentTimeMillis();
		System.out.println("transferFrom: "+ (end1 - start));
		Files.deleteIfExists(transfer.toPath);
		
		transfer.copy();
		long end2 = System.currentTimeMillis();
		System.out.println("Files copy: "+(end2 - end1));
		Files.deleteIfExists(transfer.toPath);
		transfer.ioCopy();
		long end3 = System.currentTimeMillis();
		System.out.println("original io: "+(end3 - end2));
	}

	void transferFrom() {
		try (FileChannel channelFrom = FileChannel.open(fromPath, StandardOpenOption.READ);
				FileChannel channelTo = FileChannel.open(toPath, StandardOpenOption.CREATE_NEW,
						StandardOpenOption.WRITE);) {
			channelTo.transferFrom(channelFrom, 0L, channelFrom.size());
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	void copy() {
		try {
			Files.copy(fromPath, toPath);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	void ioCopy() {
		try (InputStream is = new FileInputStream(fromPath.toFile());
				OutputStream os = new FileOutputStream(toPath.toFile());) {
			byte[] buffer = new byte[4096];
			int byteread = 0;
			while ((byteread = is.read(buffer)) != -1) {
				os.write(buffer, 0, byteread);
			}
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e1) {
			e1.printStackTrace();
		}
	}
}

　　

执行结果

transferFrom: 32726
Files copy: 33002
original io: 112975

将ioCopy方法中每次读取字节数调大十倍和百倍后结果

original io: 54743 （十倍）

original io: 93110 （百倍）

 

三、FileChannel.map 的使用

FileChannel.map 将文件按照一定大小块映射到内存区域，程序方式内存区域操作文件，适合大文件只读操作，如大文件的md5校验。

 

package com.lazy.nio;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

/**
 * 使用介绍
 * @author Lazy Gene
 *
 */
public class FileChannelMap {
	
	public static void main(String[] args) throws NoSuchAlgorithmException, IOException {
		// 计算test.tmp 的md5值，将文件分成多块计算
		Path path = Paths.get("from/test.tmp");
		File file = path.toFile();
		RandomAccessFile accessFile = new RandomAccessFile(file, "r");
		MessageDigest MD5 = MessageDigest.getInstance("MD5");
		long start = System.currentTimeMillis();
		long eachSize = 1024 * 1024L; 
		long length = file.length();
		int count = 1 + (int) (length / eachSize); //分块数量 
		long remaining = length; // 剩下的大小
		for (int i=0;i<count;i++) {
			MD5.update(accessFile.getChannel().map(FileChannel.MapMode.READ_ONLY, i * eachSize, Math.min(remaining, eachSize)));
			remaining -= eachSize;
		}
		accessFile.close();
		long end = System.currentTimeMillis();
		System.out.println("用时："+(end - start));
		System.out.println("md5: "+ new String(Hex.encodeHex(MD5.digest())));
		
		//方法二
		MessageDigest new_MD5 = MessageDigest.getInstance("MD5");
		FileInputStream in=new FileInputStream(file);
		byte[] buffer=new byte[65536];
		int rv=0;
		while((rv=in.read(buffer))>0) {
			new_MD5.update(buffer,0,rv);
		}
		long end1 = System.currentTimeMillis();
		in.close();
		System.out.println("用时："+(end1 - end));
		System.out.println("io md5: "+ new String(Hex.encodeHex(new_MD5.digest())));
	}
}　　

对于4g文件运行结果：

用时：15172
md5: f18c798ff5d450dfe4d3acdc12b621ff
用时：15811
io md5: f18c798ff5d450dfe4d3acdc12b621ff

差别不大呀，将文件调到16g，运行结果

用时：65046
md5: 0eb76b1bf69255feec7bdf4a3b5e2805
用时：62697
io md5: 0eb76b1bf69255feec7bdf4a3b5e2805

这里只所以差别不大，估计是应用map操作和操作系统的底层io实现相关，下次换成linux看看。