通过 file.createWriteStream() 流式传输到 GCP 时，获取 ESOCKETTIMEOUT、ECONNRESET 或套接字挂起大文件

Posted 2023-02-22

【中文标题】通过 file.createWriteStream() 流式传输到 GCP 时，获取 ESOCKETTIMEOUT、ECONNRESET 或套接字挂起大文件

【英文标题】：Getting ESOCKETTIMEOUT, ECONNRESET or socket hang up with large file when streaming to GCP through file.createWriteStream()

【发布时间】：2018-04-29 22:15:56

【问题描述】：

在尝试通过 Google Cloud Function 将大文件 (>50MB) 上传到 Google Cloud Storage 时，我遇到了这些异常，具体取决于我设置的选项：

在设置请求选项“永远：假”时，我得到：错误：套接字挂断 在设置请求选项 'resumable: true' 时，我得到：错误：写入 ECONNRESET 在设置请求选项 'resumable: false' 时，我得到：错误：ESOCKETTIMEDOUT with resumable: false

这是我使用的代码：

function uploadFile(bucketName, filename, data) 
  console.log("Starting uploading blob...");
  const Storage = require('@google-cloud/storage');

  console.log("Creating client...");
  const storage = new Storage();
  storage.interceptors.push(
      request: function(reqOpts) 
        reqOpts.forever = false;
        return reqOpts
      
    );

  console.log("Getting bucket " + bucketName + "...");
  const bucket = storage.bucket(bucketName);
  console.log("Creating file " + filename + "...");
  const file = bucket.file(filename);
  console.log("Creating write stream...");

  var writeStream = file.createWriteStream(
    metadata: 
      contentType: 'plain/text'
    ,
    resumable: false
  );

  writeStream.on('error', function(e)  console.error("An error occurred : " + e); );
  writeStream.on('finish', function()  console.log("Success"); );

  console.log("Initializing Streaming...");
  var bufferStream = new stream.PassThrough();
  bufferStream.end(data);
  bufferStream.pipe(writeStream);

我有什么遗漏吗？

【问题讨论】：

您运行的是哪个版本？我在1.2.1 上遇到同样的错误。

我使用的是@google-cloud/storage 1.4.0。我通过使用库“request”而不是“request-promise”解决了这个问题，效果很好！

【参考方案1】：

我通过使用库“request”（2.83.0）而不是“request-promise”解决了他的问题。这是我使用的简化代码：

const request = require('request').defaults(
    timeout: 500000,
    gzip: true,
    forever: false,
    pool: 
        maxSockets: Infinity
    
);
const Storage = require('@google-cloud/storage');
const storage = new Storage();
storage.interceptors.push(
    request: function(reqOpts) 
        reqOpts.forever = false;
        return reqOpts
    
);

/**
 * HTTP Cloud Function.
 *
 * @param Object req Cloud Function request context.
 * @param Object res Cloud Function response context.
 */
exports.exportToGCS = function exportToGCS(req, res) 
   var bucket = req.body.bucket;
   var fileName = req.body.fileName;

    try 
        console.log("Getting bucket " + bucket + "...");
        var bucket = storage.bucket(bucket);
        console.log("Creating file " + fileName + "...");
        const file = bucket.file(fileName);
        console.log("Creating writeStream...");
        var writeStream = createWriteStream(file);
        // Get the stream from a request to send out to GCS
        var options = createRequest();
        request
            .get(options)
            .pipe(writeStream);
        console.log("Streaming to Storage...");
        res.send("The export has been successfully initialized");
     catch (e) 
        console.error(e.message, e.name);
        res.status(500).send("An error occurred during the export initialization");
    
;

// Initialize the PDF write stream
function createWriteStream(file) 
    var writeStream = file.createWriteStream(
        metadata: 
            //metadata you want to set
        
    );
    writeStream .on('error', function(e)  console.error("An error occurred : " + e); );
    writeStream .on('finish', function()  console.log("Export  completed"); );
    return writeStream ;

希望对你有帮助！