使用 NodeJS 将文件上传到 Amazon S3
Posted
技术标签:
【中文标题】使用 NodeJS 将文件上传到 Amazon S3【英文标题】：Upload a file to Amazon S3 with NodeJS 【发布时间】：2015-03-17 02:49:28 【问题描述】：我在尝试将文件上传到我的 S3 存储桶时遇到了问题。一切正常，除了我的文件参数似乎不合适。我正在使用 Amazon S3 sdk 从 nodejs 上传到 s3。
这些是我的路线设置：
var multiparty = require('connect-multiparty'),
multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);
这是 items.upload() 函数：
exports.upload = function(req, res)
var file = req.files.file;
var s3bucket = new AWS.S3(params: Bucket: 'mybucketname');
s3bucket.createBucket(function()
var params =
Key: file.name,
Body: file
;
s3bucket.upload(params, function(err, data)
console.log("PRINT FILE:", file);
if (err)
console.log('ERROR MSG: ', err);
else
console.log('Successfully uploaded data');
);
);
;
将Body 参数设置为"hello" 之类的字符串可以正常工作。根据doc，Body 参数必须采用 (Buffer, Typed Array, Blob, String, ReadableStream) 对象数据。 但是，上传文件对象失败并显示以下错误消息：
[Error: Unsupported body payload object]
这是文件对象：
fieldName: 'file',
originalFilename: 'second_fnp.png',
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
headers:
'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
'content-type': 'image/png' ,
ws:
_writableState:
highWaterMark: 16384,
objectMode: false,
needDrain: true,
ending: true,
ended: true,
finished: true,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
sync: false,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
buffer: [],
errorEmitted: false ,
writable: true,
domain: null,
_events: error: [Object], close: [Object] ,
_maxListeners: 10,
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
fd: null,
flags: 'w',
mode: 438,
start: undefined,
pos: undefined,
bytesWritten: 261937,
closed: true ,
size: 261937,
name: 'second_fnp.png',
type: 'image/png'
任何帮助将不胜感激！
【问题讨论】：
您的服务器只需要存储 aws 机密即可创建 s3-signed-urls（使用 aws-sdk）。客户端检索签名的 url，然后使用该 url 在客户端进行任何上传。查看本指南：devcenter.heroku.com/articles/s3-upload-node 【参考方案1】：所以看起来这里有一些问题。根据您的帖子，您似乎正在尝试使用 connect-multiparty 中间件支持文件上传。该中间件所做的是将上传的文件写入本地文件系统，然后将req.files 设置为上传的文件。
您的路线配置看起来不错，问题似乎与您的items.upload() 函数有关。特别是这部分：
var params =
Key: file.name,
Body: file
;
正如我在答案开头提到的connect-multiparty 将文件写入本地文件系统，因此您需要打开文件并读取它，然后上传它，然后在本地文件系统上将其删除。
也就是说，您可以将方法更新为以下内容：
var fs = require('fs');
exports.upload = function (req, res)
var file = req.files.file;
fs.readFile(file.path, function (err, data)
if (err) throw err; // Something went wrong!
var s3bucket = new AWS.S3(params: Bucket: 'mybucketname');
s3bucket.createBucket(function ()
var params =
Key: file.originalFilename, //file.name doesn't exist as a property
Body: data
;
s3bucket.upload(params, function (err, data)
// Whether there is an error or not, delete the temp file
fs.unlink(file.path, function (err)
if (err)
console.error(err);
console.log('Temp File Delete');
);
console.log("PRINT FILE:", file);
if (err)
console.log('ERROR MSG: ', err);
res.status(500).send(err);
else
console.log('Successfully uploaded data');
res.status(200).end();
);
);
);
;
它的作用是从本地文件系统读取上传的文件，然后将其上传到 S3，然后删除临时文件并发送响应。
这种方法存在一些问题。首先，它的效率并不高，因为对于大文件，您将在编写之前加载整个文件。其次，这个过程不支持大文件的分段上传（我认为在你必须进行分段上传之前，截止是 5 Mb）。
我建议你使用我一直在开发的一个名为 S3FS 的模块，它提供了与 Node.JS 中的原生 FS 类似的接口，但抽象出一些细节，例如 multi -part 上传和 S3 api（以及添加一些附加功能，如递归方法）。
如果您要引入 S3FS 库，您的代码将如下所示：
var fs = require('fs'),
S3FS = require('s3fs'),
s3fsImpl = new S3FS('mybucketname',
accessKeyId: XXXXXXXXXXX,
secretAccessKey: XXXXXXXXXXXXXXXXX
);
// Create our bucket if it doesn't exist
s3fsImpl.create();
exports.upload = function (req, res)
var file = req.files.file;
var stream = fs.createReadStream(file.path);
return s3fsImpl.writeFile(file.originalFilename, stream).then(function ()
fs.unlink(file.path, function (err)
if (err)
console.error(err);
);
res.status(200).end();
);
;
这会为提供的存储桶和 AWS 凭证实例化模块，然后在存储桶不存在时创建它。然后，当请求上传文件时，我们将打开文件流并使用它将文件写入 S3 到指定路径。这将在后台处理多部分上传片段（如果需要），并且具有通过流完成的好处，因此您不必等待在开始上传之前读取整个文件。
如果您愿意，可以将代码更改为来自Promises 的回调。或者使用pipe() 方法和事件监听器来确定结束/错误。
如果您正在寻找其他方法，请查看 s3fs 的文档，如果您正在寻找其他方法或遇到问题，请随时提出问题。
【讨论】：
我有一个存储桶和一个文件夹。如何为上传到此文件夹的对象指定 ACL。我已将文件夹权限更改为“公开”，但每次上传新对象时它都不是公开的。我在 S3FS 构造函数中尝试了 acl: "public-read" 但它不起作用。 如果您在编写对象时使用的是 S3FS，则可以指定 ACL。检查此链接：docs.aws.amazon.com/AWSjavascriptSDK/latest/AWS/… 否则，您需要在 S3FS 之外上传文件时指定策略。 req.file.originalname 对我有用，而不是 req.file.originalFilename。【参考方案2】：我发现以下是一个可行的解决方案::
npm install aws-sdk
安装 aws-sdk 后，在需要的地方使用以下代码替换值。
var AWS = require('aws-sdk');
var fs = require('fs');
var s3 = new AWS.S3();
// Bucket names must be unique across all S3 users
var myBucket = 'njera';
var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data)
//for Video file
//fs.readFile('demo.avi', function (err, data)
//for image file
fs.readFile('demo.jpg', function (err, data)
if (err) throw err;
params = Bucket: myBucket, Key: myKey, Body: data ;
s3.putObject(params, function(err, data)
if (err)
console.log(err)
else
console.log("Successfully uploaded data to myBucket/myKey");
);
);
我在这里找到了关于这个主题的完整教程，以防你在寻找参考资料::
How to upload files (text/image/video) in amazon s3 using node.js
【讨论】：
在哪里设置 s3 密钥和秘密？ 在实例化新的 AWS.S3() 之前，您需要添加如下内容：AWS.config.update( accessKeyId: 'someKey', secretAccessKey: 'secondKey' );
如果我想上传本地系统中的一些文件？怎么做？？【参考方案3】：
或使用承诺：
const AWS = require('aws-sdk');
AWS.config.update(
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
region: 'region'
);
let params =
Bucket: "yourBucketName",
Key: 'someUniqueKey',
Body: 'someFile'
;
try
let uploadPromise = await new AWS.S3().putObject(params).promise();
console.log("Successfully uploaded data to bucket");
catch (e)
console.log("Error uploading data: ", e);
【讨论】：
【参考方案4】：var express = require('express')
app = module.exports = express();
var secureServer = require('http').createServer(app);
secureServer.listen(3001);
var aws = require('aws-sdk')
var multer = require('multer')
var multerS3 = require('multer-s3')
aws.config.update(
secretAccessKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
accessKeyId: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
region: 'us-east-1'
);
s3 = new aws.S3();
var upload = multer(
storage: multerS3(
s3: s3,
dirname: "uploads",
bucket: "Your bucket name",
key: function (req, file, cb)
console.log(file);
cb(null, "uploads/profile_images/u_" + Date.now() + ".jpg"); //use
Date.now() for unique file keys
)
);
app.post('/upload', upload.single('photos'), function(req, res, next)
console.log('Successfully uploaded ', req.file)
res.send('Successfully uploaded ' + req.file.length + ' files!')
)
【讨论】：
虽然这段代码 sn-p 可以解决问题，including an explanation 确实有助于提高您的帖子质量。请记住，您正在为将来的读者回答问题，而这些人可能不知道您的代码建议的原因。也请尽量不要用解释性的 cmets 挤满你的代码，这会降低代码和解释的可读性！【参考方案5】：将文件上传到 AWS s3 并发送 url 以响应访问文件。
Multer 是一个用于处理 multipart/form-data 的 node.js 中间件，主要用于上传文件。它被写在 busboy 的顶部以获得最大的效率。检查这个 npm 模块here。
当您发送请求时，请确保标头的 Content-Type 为 multipart/form-data。 我们在响应中发送文件位置，这将提供 url，但如果您想访问该 url，请将存储桶公开，否则您将无法访问它。
上传.router.js
const express = require('express');
const router = express.Router();
const AWS = require('aws-sdk');
const multer = require('multer');
const storage = multer.memoryStorage()
const upload = multer(storage: storage);
const s3Client = new AWS.S3(
accessKeyId: 'your_access_key_id',
secretAccessKey: 'your_secret_access_id',
region :'ur region'
);
const uploadParams =
Bucket: 'ur_bucket_name',
Key: '', // pass key
Body: null, // pass file body
;
router.post('/api/file/upload', upload.single("file"),(req,res) =>
const params = uploadParams;
uploadParams.Key = req.file.originalname;
uploadParams.Body = req.file.buffer;
s3Client.upload(params, (err, data) =>
if (err)
res.status(500).json(error:"Error -> " + err);
res.json(message: 'File uploaded successfully','filename':
req.file.originalname, 'location': data.Location);
);
);
module.exports = router;
app.js
const express = require('express');
const app = express();
const router = require('./app/routers/upload.router.js');
app.use('/', router);
// Create a Server
const server = app.listen(8080, () =>
console.log("App listening at 8080");
)
【讨论】：
【参考方案6】：感谢 David，因为他的解决方案帮助我提出了将多部分文件从 Heroku 托管站点上传到 S3 存储桶的解决方案。我使用强大的来处理传入的表单和 fs 来获取文件内容。希望对您有所帮助。
api.service.ts
public upload(files): Observable<any>
const formData: FormData = new FormData();
files.forEach(file =>
// create a new multipart-form for every file
formData.append('file', file, file.name);
);
return this.http.post(uploadUrl, formData).pipe(
map(this.extractData),
catchError(this.handleError));
server.js
app.post('/api/upload', upload);
app.use('/api/upload', router);
上传.js
const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');
module.exports = function upload(req, res)
var form = new IncomingForm();
const bucket = new AWS.S3(
signatureVersion: 'v4',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: 'us-east-1'
);
form.on('file', (field, file) =>
const fileContent = fs.readFileSync(file.path);
const s3Params =
Bucket: process.env.AWS_S3_BUCKET,
Key: 'folder/' + file.name,
Expires: 60,
Body: fileContent,
ACL: 'public-read'
;
bucket.upload(s3Params, function(err, data)
if (err)
throw err;
console.log('File uploaded to: ' + data.Location);
fs.unlink(file.path, function (err)
if (err)
console.error(err);
console.log('Temp File Delete');
);
);
);
// The second callback is called when the form is completely parsed.
// In this case, we want to send back a success status code.
form.on('end', () =>
res.status(200).json('upload ok');
);
form.parse(req);
上传图片.component.ts
import Component, OnInit, ViewChild, Output, EventEmitter, Input from '@angular/core';
import ApiService from '../api.service';
import MatSnackBar from '@angular/material/snack-bar';
@Component(
selector: 'app-upload-image',
templateUrl: './upload-image.component.html',
styleUrls: ['./upload-image.component.css']
)
export class UploadImageComponent implements OnInit
public files: Set<File> = new Set();
@ViewChild('file', static: false ) file;
public uploadedFiles: Array<string> = new Array<string>();
public uploadedFileNames: Array<string> = new Array<string>();
@Output() filesOutput = new EventEmitter<Array<string>>();
@Input() CurrentImage: string;
@Input() IsPublic: boolean;
@Output() valueUpdate = new EventEmitter();
strUploadedFiles:string = '';
filesUploaded: boolean = false;
constructor(private api: ApiService, public snackBar: MatSnackBar,)
ngOnInit()
updateValue(val)
this.valueUpdate.emit(val);
reset()
this.files = new Set();
this.uploadedFiles = new Array<string>();
this.uploadedFileNames = new Array<string>();
this.filesUploaded = false;
upload()
this.api.upload(this.files).subscribe(res =>
this.filesOutput.emit(this.uploadedFiles);
if (res == 'upload ok')
this.reset();
, err =>
console.log(err);
);
onFilesAdded()
var txt = '';
const files: [key: string]: File = this.file.nativeElement.files;
for (let key in files)
if (!isNaN(parseInt(key)))
var currentFile = files[key];
var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
var iFileSize = currentFile.size;
if (!(sFileExtension === "jpg"
|| sFileExtension === "png")
|| iFileSize > 671329)
txt = "File type : " + sFileExtension + "\n\n";
txt += "Size: " + iFileSize + "\n\n";
txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
alert(txt);
return false;
this.files.add(files[key]);
this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
this.uploadedFileNames.push(files[key].name);
if (this.IsPublic && this.uploadedFileNames.length == 1)
this.filesUploaded = true;
this.updateValue(files[key].name);
break;
else if (!this.IsPublic && this.uploadedFileNames.length == 3)
this.strUploadedFiles += files[key].name;
this.updateValue(this.strUploadedFiles);
this.filesUploaded = true;
break;
else
this.strUploadedFiles += files[key].name + ",";
this.updateValue(this.strUploadedFiles);
addFiles()
this.file.nativeElement.click();
openSnackBar(message: string, action: string)
this.snackBar.open(message, action,
duration: 2000,
verticalPosition: 'top'
);
上传图片.component.html
<input type="file" #file style="display: none" (change)="onFilesAdded()" multiple />
<button mat-raised-button color="primary"
[disabled]="filesUploaded" (click)="$event.preventDefault(); addFiles()">
Add Files
</button>
<button class="btn btn-success" [disabled]="uploadedFileNames.length == 0" (click)="$event.preventDefault(); upload()">
Upload
</button>
【讨论】：
这篇文章很棒，感谢您的提示！【参考方案7】：上传 CSV/Excel
const fs = require('fs');
const AWS = require('aws-sdk');
const s3 = new AWS.S3(
accessKeyId: XXXXXXXXX,
secretAccessKey: XXXXXXXXX
);
const absoluteFilePath = "C:\\Project\\test.xlsx";
const uploadFile = () =>
fs.readFile(absoluteFilePath, (err, data) =>
if (err) throw err;
const params =
Bucket: 'testBucket', // pass your bucket name
Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
Body: data
;
s3.upload(params, function (s3Err, data)
if (s3Err) throw s3Err
console.log(`File uploaded successfully at $data.Location`);
debugger;
);
);
;
uploadFile();
【讨论】：
【参考方案8】：为我工作:)
const fileContent = fs.createReadStream(`$fileName`);
return new Promise(function (resolve, reject)
fileContent.once('error', reject);
s3.upload(
Bucket: 'test-bucket',
Key: `$fileName + '_' + Date.now().toString()`,
ContentType: 'application/pdf',
ACL: 'public-read',
Body: fileContent
,
function (err, result)
if (err)
reject(err);
return;
resolve(result.Location);
);
);```
【讨论】：
您能提供更多解释吗？此代码将如何帮助 OP？【参考方案9】：使用 aws SDK v3
npm install @aws-sdk/client-s3
上传代码
import S3Client, PutObjectCommand from "@aws-sdk/client-s3";
/**
* advisable to save your AWS credentials and configurations in an environmet file. Not inside the code
* AWS lib will automatically load the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if available in your environment
*/
const s3Client = new S3Client( region: process.env.AWS_S3_REGION );
/**
* upload a file
* @param file the file object to be uploaded
* @param fileKey the fileKey. could be separated with '/' to nest the file into a folder structure. eg. members/user1/profile.png
*/
export function uploadFile(file, fileKey)
s3Client.send(new PutObjectCommand(
Bucket: process.env.MY_AWS_S3_BUCKET,
Key: fileKey,
Body: file
));
如果你想下载
import GetObjectCommand from "@aws-sdk/client-s3";
/**
* download a file from AWS and send to your rest client
*/
app.get('/download', function(req, res, next)
var fileKey = req.query['fileKey'];
var bucketParams =
Bucket: 'my-bucket-name',
Key: fileKey,
;
res.attachment(fileKey);
var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
// for TS you can add: if (fileStream.Body instanceof Readable)
fileStream.Body.pipe(res)
);
【讨论】：
以上是关于使用 NodeJS 将文件上传到 Amazon S3的主要内容，如果未能解决你的问题，请参考以下文章
如何使用 POSTMAN 应用程序将文件上传到 Amazon S3