javascript bulk inset update

Posted 2020-12-11 tonggc1668

 

single bson is 1K

bulk insert 1k 96ms
update 1k without index bulkwrite 877ms bulkupdate 910ms
update 1k with index bulkwrite 107ms bulkupdate 120ms

bulk insert 10k 592ms
update 10k with index bulkwrite 1045ms bulkupdate 1200ms

bulk insert 100k 30025ms 25834ms
update 1k with index bulkwrite unorderd 109ms
update 10k with index bulkwrite unorderd 998ms
update 10k with index bulkwrite 1252ms bulkupdate 1555ms
update 100k with index bulkwrite unorderd 11654ms

 

 

 

 

try {
   db.restaurant.updateMany(
       { "name" : "Pizza Rat‘s Pizzaria" },
       { $inc: { "violations" : 3}, $set: { "Closed" : true } },
       { w: "majority", wtimeout: 100 }
   );
} catch (e) {
   print(e);
}

try {
   db.CollateralRepresentationBulkTest.bulkWrite(
      [
         { updateMany :
            {
               "filter" : { "uicid" : "170484_SMCP_1001059_20200131_20200302" },
               "update" : { $set : { "xipEnvId" : "test2" } }
            }
         },
         { updateMany :
            {
               "filter" : { "uicid" : "170484_SMCP_100159930_20200131_20200302" },
               "update" : { $set : { "xipEnvId" : "test2" } }
            }
         }
      ]
         //,{ writeConcern : { w : "majority", wtimeout : 20 }, ordered : false }
   );
} catch (e) {
   print(e);
}

 var ops = [];

  

try {
var trades = db.CollateralRepresentationBulkTest.find({},{ uicid: 1,  _id: 0 }).limit(1000);
while(trades.hasNext()){
                    
                       ops.push(
                       { updateMany :
            {
               "filter" : { "uicid" : trades.next().uicid },
               "update" : { $set : { "xipEnvId" : "test" } }
            }
         }
                       );
                    }
                    var startTime = (new Date()).getTime();
   db.CollateralRepresentationBulkTest.bulkWrite(
      ops
      //,{ writeConcern : { w : "majority", wtimeout : 100 }, ordered : false }
   );
   var runTime = (new Date()).getTime()-startTime;
                    print("Finished, runTime is "+runTime+"ms");
} catch (e) {
   print(e);
}

tmDb = db.getSiblingDB(‘timemachine‘);




         try{
                    var trades = tmDb.CollateralRepresentationBulkTest.find({}).limit(1000);
                    var bulk = tmDb.CollateralRepresentationBulkTest.initializeUnorderedBulkOp();
                    var startTime = (new Date()).getTime();
                    while(trades.hasNext()){
                    
                       bulk.find( { uicid: trades.next().uicid } ).update( { $set: { xipEnvId: "test" } } );
                    }
                    bulk.execute();
                    
                    var runTime = (new Date()).getTime()-startTime;
                    print("Finished, runTime is "+runTime+"ms");
                 }catch (e){
                   print (e);
                 }
                 

tmDb = db.getSiblingDB(‘timemachine‘);


db.CollateralRepresentationBulkTest.drop();

         try{
                    var trades = tmDb.CollateralRepresentation.find({}).limit(1000);
                    var bulk = tmDb.CollateralRepresentationBulkTest.initializeUnorderedBulkOp();
                    
                    while(trades.hasNext()){
                       bulk.insert(trades.next());
                    }
                    var startTime = (new Date()).getTime();
                    bulk.execute();
                    var runTime = (new Date()).getTime()-startTime;
                    print("Finished, runTime is "+runTime+"ms");
                    tmDb.CollateralRepresentationBulkTest.createIndex({"uicid":1, },{background : true})
                 }catch (e){
                   print (e);
                 }