如何将传入数据（NDJ）从一个API传递到CSV格式的另一个API，而不创建文件？

Posted 2023-01-05

我有一个要以NDJ格式发送响应的API，如下所示，并希望将响应作为csv（逗号分隔的值）传递给另一个API

"country":"Germany","os":"Windows 7","utm_campaign":"(none)","utm_medium":"direct / organic","from_mobile":false,"base_browser":"Chrome","utm_term":"(none)","domain":"furnitureshop4u-2.myshopify.com","browser":"Chrome 76.0.3809","repeat":true,"location":"Meinerzhagen - North Rhine-Westphalia, Germany","base_os":"Windows","utm_source":"(none)""country":"Germany","os":"Windows 7","utm_campaign":"(none)","utm_medium":"direct / organic","from_mobile":false,"base_browser":"Chrome","utm_term":"(none)","domain":"furnitureshop4u-2.myshopify.com","browser":"Chrome 76.0.3809","repeat":true,"location":"Meinerzhagen - North Rhine-Westphalia, Germany","base_os":"Windows","utm_source":"(none)"

我试图缓冲来自源API的响应，并使用memorystream对其进行了更改，并将其转换为csv（逗号分隔的值）。现在我很震惊，我不知道如何将数据传输到目标API。

var body = "request_json" :JSON.stringify(options);
    var apicall = 
        url: API,
        method: 'post',
        responseType: 'stream',
        headers: 
            'Content-Type': 'application/json',
        ,
        data: body
    
    axios(apicall).then((res) =>
       var writer = new MemoryStream(null, 
           readable : true
       ) // using memorystream to stream the buffer
       res.data.pipe(writer); // writing data to a writer
       writer.on('data',function(dat)
           console.log(dat); //getting buffer
           var e = dat.toString();
           console.log(e);
           var x = JSON.parse(e);
           var y = [x];
           console.log(y); // converted to JSON
           var keys = Object.keys(x);
           var csv = json2csv.convertArrayOfObjects(y, keys) //converting to csv comma separated string
           console.log(csv,'Final');
       );       
       writer.on('end', function()
        //console.log('updated ---- ',data);
       );

    ).catch((e)=>
        console.log(e);
    )

我只想将数据以CSV格式从源API传递到目标API，而不将数据存储到csv文件中。

答案

由于我们已经使用memorystream将传入的数据转换为流，因此我们可以如下将其转换并直接流至目标API

const axios = require('axios')
const FormData = require('form-data')
const MemoryStream = require('memorystream')
const  AsyncParser  = require('json2csv')

const body =  request_json: JSON.stringify(options) 
const apicall = 
    url: API,
    method: 'post',
    responseType: 'stream',
    headers: 
        'Content-Type': 'application/json'
    ,
    data: body

axios(apicall)
.then(res => 
    var writer = new MemoryStream(null, 
        readable: true
    )
    const transformOpts =  encoding: 'utf-8' 
    const asyncParser = new AsyncParser(transformOpts)
    asyncParser.fromInput(res.data).toOutput(writer)
    const formData = new FormData()
    formData.append('source', 'sample_table')
    formData.append('file', writer, filename:file,contentType:'text/csv')
    axios
        .post(finalAPI, formData, 
            headers: formData.getHeaders(),
            maxContentLength: 2 * 1024 * 1024 * 1024
        )
        .then(function(a) 
            console.log('SUCCESS!!', a)
        )
        .catch(function(e) 
            console.log('FAILURE!!', e)
        )
)
.catch(e => 
    console.log(e)
)