获取s3对象元数据然后创建流

Posted 2021-04-19

我正在从s3下载一个对象并从中创建一个读取流对象来处理视频：

s3.getObject(params).createReadStream()

但是，当我通过访问其'metadata'属性获取对象时，我需要从中获取元数据：

s3.getObject()

我怎么会：

1. 通过s3.getObject()获取对象，从元数据属性中获取元数据，然后将其转换为读取流？ var stream = fs.createReadStream(response);不工作 - 输入必须是一个字符串

- 要么 -

2. 通过s3.getObject().createReadStream()获取流，并从流中提取元数据？ 据我所知，元数据不会在流中传递。

---

告诉我，如果我的假设是错误的，但我目前仍然坚持这两个需求：

• 获取元数据
• 使它成为一个流

答案

您可以通过请求的httpHeaders事件获取元数据。

let fs = require('fs')
let aws = require('aws-sdk')
let s3 = new aws.S3()

let request = s3.getObject({
    Bucket: 'my-bucket',
    Key: 'my-key'
})

let stream

request.on('httpHeaders', (statusCode, httpHeaders) => {
    // object metadata is represented by any header in httpHeaders starting with 'x-amz-meta-'
    // you can use the stream object that this point
    stream.pipe(fs.createWriteStream('./somepath'))
    stream.on('end', () => { 
        console.log('were done')
    })
})

stream = request.createReadStream()

或者你也可以调用s3.headObject获取元数据而不下载对象，然后使用s3.getObject下载对象

另一答案

所以我找到了一个解决方案。这适用于10 MB以下的大多数文件。如果它们大于缓冲流，则在写入文件之前结束缓冲流。我已经尝试将bufferStream.end放在on finish函数中，但后来我的回调没有通过....

function download(s3Event, srcKey, cb){

  console.log('Starting download');

  s3.getObject({
      Bucket: s3Event.bucket.name,
      Key: srcKey
  }, cb);
}



function writeToFile(data, cb){

  var dlFile = path.join(tempDir, 'download');

  console.log('data = ', data);
  console.log('data.Body = ', data.Body);
  var stream = bufferStream.pipe(fs.createWriteStream(dlFile)).on('finish', function () {
     console.log('finished writing stream');
     cb(null, data);
  });

  bufferStream.end(data.Body);
}


exports.handler = function(event, context) {

// Read options from the event.

console.log("Reading options from event:
", util.inspect(event, {depth: 5}));

var s3Event = event.Records[0].s3;
var srcKey = decodeURIComponent(s3Event.object.key);
var keyPrefix = srcKey.replace(/.[^/.]+$/, '');         
var dstBucket = "jump-lambda";


async.waterfall([

    function (cb){
        download(s3Event, srcKey, cb);
    },

    function (data, cb){
        writeToFile(data, cb);
    },

    function (data, cb){
        fluentffmpegProcess(data, cb);
    },

    function (data, cb){
        transform(data, cb);
    },

    function (data, buffer, cb){
        thumbnailUpload(data, buffer, dstBucket, keyPrefix, cb);
    },

    function (data, cb){
        updateParse(data, srcKey, keyPrefix, cb);
    },

],

    function (err) {
        if (err) {
            console.error(
                'Unable to convert video to scene object, with error code: ' + err.description
            );
        } else {
            console.log(
                'Successfully created scene object, updated venue, and created thumbnail'
            );
        }
    }
);
};