Spring Boot 集成 AmazonS3 存储服务教程

Posted 2022-12-07 壹言

原文链接：https://www.changxuan.top/?p=963

环境准备

1. Java环境：JDK6以上版本
2. SpringBoot 2.2.5
3. 存储服务的 accessKey、secreKey 以及 地址:端口

使用

1. 引入依赖

在 pom.xml 文件中添加下列依赖：

<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk</artifactId>
    <version>1.11.433</version>
</dependency>

2. 初始化

> 简单模式

String accessKey = "your-accesskey";
String secretKey = "your-secretKey";

AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 conn = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("endpoint","region"))
                .build();

示例：

String accessKey = "TPDDEA5PCT9C8RUPKAWW";
String secretKey = "ryesPMSiSfOzaP1TkFe9TIOBnxpdrA2sw6isFwDZ";

AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 conn = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new   AwsClientBuilder.EndpointConfiguration("http://10.200.100.37:7480","region"))
                .build();

> 配置模式

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;

String accessKey = "your-accesskey";
String secretKey = "your-secretKey ";
AWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
ClientConfiguration conf = new ClientConfiguration();
// 设置AmazonS3使用的最大连接数
conf.setMaxConnections(200);
// 设置socket超时时间
conf.setSocketTimeout(10000);
// 设置失败请求重试次数
conf.setMaxErrorRetry(2);
// 如果要用https协议，请加上下面语句
conf.setProtocol(Protocol.HTTPS);

//AmazonS3 s3Client = new AmazonS3Client(credentials,clientConfiguration);
//s3Client.setEndpoint(endPoint);
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
	            .withCredentials(new AWSStaticCredentialsProvider(credentials))
	            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("endpoint","region"))
            .withClientConfiguration(conf)
            .build();//endpoint,region请指定为NOS支持的

示例：

Amazon3 是线程安全的，因此可以将其注入到 Spring 容器中进行管理。

@AmazonS3Config.java

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @Author: ChangXuan
 * @Decription: 初始化AmazonS3实例
 * @Date: 17:33 2020/6/3
 **/
@Configuration
@EnableConfigurationProperties(UploadConfig.class)
public class AmazonS3Config 

    @Autowired
    private UploadConfig uploadConfig;

    @Bean(name = "amazonS3")
    public AmazonS3 getAmazonS3()
        String accessKey = uploadConfig.getCeph().getAccessKey();
        String secretKey = uploadConfig.getCeph().getSecretKey();
        AWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
        ClientConfiguration conf = new ClientConfiguration();
        // 设置AmazonS3使用的最大连接数
        conf.setMaxConnections(uploadConfig.getCeph().getAmazonS3MaxConnections());
        // 设置socket超时时间
        conf.setSocketTimeout(uploadConfig.getCeph().getAmazonS3SocketTimeout());
        // 设置失败请求重试次数
        conf.setMaxErrorRetry(uploadConfig.getCeph().getAmazonS3MaxErrorRetry());
        // 设置协议
        if (!"blank".equals(uploadConfig.getCeph().getAmazonS3Protocol()))
            switch (uploadConfig.getCeph().getAmazonS3Protocol())
                case "https":
                    conf.setProtocol(Protocol.HTTPS);
                    break;
                case "http":
                    conf.setProtocol(Protocol.HTTP);
                    break;
                default:
                    break;
            
        
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uploadConfig.getCeph().getHosts().get(0),uploadConfig.getCeph().getRegion()))
                .withClientConfiguration(conf)
                .build();
        checkAndCreateBucket(s3Client);
        return s3Client;
    


    @Bean(name = "transferManager")
    public TransferManager getTransferManager()
        return TransferManagerBuilder.standard().withS3Client(getAmazonS3()).build();
    


    /**
     * 检查桶是否存在，不存在则创建创建
     * @param s3Client
     */
    private void checkAndCreateBucket(AmazonS3 s3Client)
        boolean exists = s3Client.doesBucketExistV2(uploadConfig.getCeph().getBucketName());
        if (!exists)
            CreateBucketRequest request = new CreateBucketRequest(uploadConfig.getCeph().getBucketName());
            s3Client.createBucket(request);
        
    

@UploadConfig.java

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;

/**
 * @Author: ChangXuan
 * @Decription: 上传 配置
 * @Date: 17:25 2020/6/3
 **/
@ConfigurationProperties(prefix = "upload")
@Data
public class UploadConfig 
    private CephConfig ceph;

@CephConfig.java

import com.google.common.collect.Lists;
import lombok.Data;

import java.util.List;

/**
 * @Author: ChangXuan
 * @Decription:配置
 * @Date: 17:28 2020/6/3
 **/
@Data
public class CephConfig 

    /**
     * 集群ip:port
     */
    private List<String> hosts = Lists.newArrayList();

    private String accessKey;

    private String secretKey;

    private String region;

    /**
     * 连接协议
     */
    private String amazonS3Protocol;

    /**
     * 失败请求重试次数
     */
    private Integer amazonS3MaxErrorRetry;

    /**
     * 超时时间
     */
    private Integer amazonS3SocketTimeout;

    /**
     * 最大连接数
     */
    private Integer amazonS3MaxConnections;

    /**
     * 桶名称
     */
    private String bucketName;

@application.yml

upload:
  ceph:
    bucketName: ic-storage-dev
    region: region
    hosts:
      - "10.200.100.37:7480"
      - "10.200.100.38:7480"
      - "10.200.100.39:7480"
    accessKey: TPDDEA5PCT9C8RUPKAWW
    secretKey: ryesPMSiSfOzaP1TkFe9TIWQnxpdrA2sw6isFwDZ
    amazonS3MaxConnections: 200
    amazonS3SocketTimeout: 10000
    amazonS3MaxErrorRetry: 2
    amazonS3Protocol: http

3. 依赖注入

@Autowired
private AmazonS3 amazonS3;

//或

@Autowired
private TransferManager transferManager;

4. 文件操作

文件上传

1. 直接内容上传

//要上传文件内容
String content = "Object content";
try 
   amazonS3.putObject("your-bucketname","your-objectname",content);
catch (Exception e)
   System.out.println(e.getMessage());

2. 本地文件普通上传

对于小对象可以使用putObject接口进行上传，putObject上传支持的最大文件大小为100M，如果上传大于100M的文件需要使用分块上传。本地文件普通上传的示例代码如下：

//要上传文件的路径
String filePath = "your-local-file-path";
try 
   amazonS3.putObject("your-bucketname","your-objectname", new File(filePath));
catch (Exception e)
   System.out.println(e.getMessage());

3. 上传文件时设置文件元数据信息

String filePath = "your-local-file-path";
ObjectMetadata objectMetadata = new ObjectMetadata();
//设置Content-Type
objectMetadata.setContentType("application/xml");
//设置标准http消息头（元数据）
objectMetadata.setHeader("Cache-Control", "no-cache");
//设置用户自定义元数据信息
Map<String, String> userMeta = new HashMap<String, String>();
userMeta.put("ud", "test");
objectMetadata.setUserMetadata(userMeta);
PutObjectRequest putObjectRequest = new PutObjectRequest("your-bucketname","your-objectname", new File(filePath));
putObjectRequest.setMetadata(objectMetadata);
amazonS3.putObject(putObjectRequest);

4. 流式上传

try 
   ObjectMetadata objectMetadata = new ObjectMetadata();
   //设置流的长度，您还可以设置其他文件元数据信息
   objectMetadata.setContentLength(streamLength);
   amazonS3.putObject("your-bucketname","your-objectname", inputStream, objectMetadata)
catch (Exception e)
   System.out.println(e.getMessage());

文件下载

1. 流式下载

S3Object fileObject = amazonS3.getObject("your-bucketname","your-objectname");
//可以通过getObjectMetadata方法获取对象的ContentType等元数据信息
String contentType = fileObject.getObjectMetadata().getContentType();
//流式获取文件内容
InputStream in = fileObject.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
while (true) 
  String line;
  try 
    line = reader.readLine();
    if (line == null) break;
    System.out.println("\\n" + line);
   catch (IOException e) 
    e.printStackTrace();
  

try 
  reader.close();
 catch (IOException e) 
  e.printStackTrace();

2. 下载到本地文件

String destinationFile = "your-local-filepath";
GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
ObjectMetadata objectMetadata = amazonS3.getObject(getObjectRequest, new File(destinationFile));

3. Range 下载

GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
getObjectRequest.setRange(0, 100);
S3Object nosObject = amazonS3.getObject(getObjectRequest);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
while (true) 
  String line;
  try 
    line = reader.readLine();
    if (line == null) break;
      System.out.println("\\n" + line);
   catch (IOException e) 
    e.printStackTrace();
  

try 
  reader.close();
 catch (IOException e) 
  e.printStackTrace();

文件管理

判断文件是否存在

您可以通过AmazonS3.doesObjectExist判断文件是否存在。

boolean isExist = amazonS3.doesObjectExist("your-bucketname","your-objectname");

文件删除

您可以通过AmazonS3.deleteObject删除单个文件

amazonS3.deleteObject("your-bucketname","your-objectname");

您还可以通过AmazonS3.deleteObjects一次删除多个文件

try 
    DeleteObjectsResult result = amazonS3.deleteObjects(deleteObjectsRequest);
    List<DeletedObject>  deleteObjects = result.getDeletedObjects();
    //print the delete results
    for (DeletedObject items: deleteObjects)
      System.out.println(items.getKey());
    
// 部分对象删除失败
 catch (MultiObjectDeleteException e)  
    List<DeleteError> deleteErrors = e.getErrors();
    for (DeleteError error : deleteErrors) 
        System.out.println(error.getKey());
    
 catch (AmazonServiceException  e) 
        //捕捉服务器异常错误
 catch (AmazonClientException ace) 
       //捕捉客户端错误

获取文件元数据信息

您可以通过AmazonS3.getObjectMetadata获取文件元数据信息

amazonS3.getObjectMetadata("your-bucketname","your-objectname");

文件复制(copy)

您可以通过AmazonS3.copyObject接口实现文件拷贝功能。

amazonS3.copyObject("source-bucket", "source-object", "dst-bucket", "dst-object");

列举桶内文件

您可以通过AmazonS3.listObjects列出桶里的文件。listObjects接口如果调用成功，会返回一个ObjectListing对象，列举的结果保持在该对象中。

ObjectListing的具体信息如下表所示：

方法	含义
List getObjectSummaries()	返回的文件列表（包含文件的名称、Etag的元数据信息）
String getPrefix()	本次查询的文件名前缀
String getDelimiter()	文件分界符
String getMarker()	这次List Objects的起点
int getMaxKeys()	响应请求内返回结果的最大数目
String getNextMarker()	下一次List Object的起点
boolean isTruncated()	是否截断，如果因为设置了limit导致不是所有的数据集都返回，则该值设置为true
List getCommonPrefixes()	如果请求中指定了delimiter参数，则返回的包含CommonPrefixes元素。该元素标明以delimiter结尾，并有共同前缀的对象的集合

AmazonS3.listObjects接口提供两种调用方式：简单列举、通过ListObjectsRequest列举

简单列举

简单列举只需指定需要列举的桶名，最多返回100条对象记录，建议桶内对象数较少时（小于100）使用。

ObjectListing objectListing = amazonS3.listObjects("your-bucketname");
List<S3ObjectSummary> sums = objectListing.getObjectSummaries();
for (S3ObjectSummary s : sums) 
  System.out.println("\\t" + s.getKey());

通过ListObjectsRequest列举

您还可以通过设置ListObjectsReques参数实现各种灵活的查询功能。ListObjectsReques的可设置的参数如下：

设置方法	作用
setPrefix(String prefix)	限定返回的object key必须以prefix作为前缀
setDelimiter(String delimiter)	是一个用于对Object名字进行分组的字符。所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组元素——CommonPrefixes
setMarker(String marker)	字典序的起始标记，只列出该标记之后的部分
setMa 以上是关于Spring Boot 集成 AmazonS3 存储服务教程的主要内容，如果未能解决你的问题，请参考以下文章 Spring Boot 第七章 Spring Boot 集成 Dubbo spring-boot 集成 rabbitmq spring-boot 集成 rabbitmq Spring Boot(十八)：使用Spring Boot集成FastDFS Spring Boot集成Mybatis及通用Mapper Shiro：Spring-boot如何集成Shiro（上）