javascript Google Cloud函数用于在pub / sub和pub / sub到Big查询中发布数据

Posted 2021-05-04

//gcloud beta functions deploy  --trigger-topic data --stage-bucket XXXXXXX-ml-mlengine --project XXXXXX subscribeDataFlow 
//gcloud beta functions deploy  --trigger-http --stage-bucket XXXXXXX-ml-mlengine --project XXXXXX publishData

var pubsub = require('@google-cloud/pubsub')();
const bigquery = require('@google-cloud/bigquery')();


exports.publishData = function(req, res) {
  var topic = pubsub.topic('data');
  var publisher = topic.publisher();
  var key = req.body.key || "test10";
  var number = req.body.number || 100;
  
  for (var i = number - 1; i >= 0; i--) {
      var data  = new Buffer(JSON.stringify({"key": key, "value": key+'--'+i}));
      publisher.publish(data).then(function(messageId) {
      console.log("correcto: "+messageId);
  });
  }
  res.status(200).json({"result":"ok", "key": key, "number": number});
  
}

exports.subscribeDataFlow = function subscribe(event, callback) {
     // The Cloud Pub/Sub Message object.
     const pubsubMessage = event.data;
     console.log('Procesado --> '+ Buffer.from(pubsubMessage.data, 'base64').toString());
     var rawData =  Buffer.from(pubsubMessage.data, 'base64').toString();
     var data = JSON.parse(rawData);
     var myDataset = bigquery.dataset('dataflow_example');
     var tabla_uno = myDataset.table('dataflow_example_gcf');
     // Import data into a table.
     tabla_uno.insert({
      key: data.key+"-gcf",
      value: data.value
     }, function(err, job) {});
     callback();
 };