mongodb通过mapreduce实现数据base64编解码

Posted 2021-04-19 BackEndDeveloper

创建base64函数

编写base64函数（编码）

function base64_encode(str){	function Base64() { 
  _keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; 
  this.encode = function (input) { 
    var output = ""; 
    var chr1, chr2, chr3, enc1, enc2, enc3, enc4; 
    var i = 0; 
    input = _utf8_encode(input); 
    while (i < input.length) { 
      chr1 = input.charCodeAt(i++); 
      chr2 = input.charCodeAt(i++); 
      chr3 = input.charCodeAt(i++); 
      enc1 = chr1 >> 2; 
      enc2 = ((chr1 & 3) << 4) | (chr2 >> 4); 
      enc3 = ((chr2 & 15) << 2) | (chr3 >> 6); 
      enc4 = chr3 & 63; 
      if (isNaN(chr2)) { 
        enc3 = enc4 = 64; 
      } else if (isNaN(chr3)) { 
        enc4 = 64; 
      } 
      output = output + 
      _keyStr.charAt(enc1) + _keyStr.charAt(enc2) + 
      _keyStr.charAt(enc3) + _keyStr.charAt(enc4); 
    } 
    return output; 
  }   
  _utf8_encode = function (string) { 
    string = string.replace(/\r\n/g,"\n"); 
    var utftext = ""; 
    for (var n = 0; n < string.length; n++) { 
      var c = string.charCodeAt(n); 
      if (c < 128) { 
        utftext += String.fromCharCode(c); 
      } else if((c > 127) && (c < 2048)) { 
        utftext += String.fromCharCode((c >> 6) | 192); 
        utftext += String.fromCharCode((c & 63) | 128); 
      } else { 
        utftext += String.fromCharCode((c >> 12) | 224); 
        utftext += String.fromCharCode(((c >> 6) & 63) | 128); 
        utftext += String.fromCharCode((c & 63) | 128); 
      } 
    } 
    return utftext; 
  }  
}
base_64_functions = new Base64();return base_64_functions.encode(str)
}

编写base64函数（解码）

function base64_decode(str){	function Base64() { 
  _keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";  
  this.decode = function (input) { 
    var output = ""; 
    var chr1, chr2, chr3; 
    var enc1, enc2, enc3, enc4; 
    var i = 0; 
    input = input.replace(/[^A-Za-z0-9\+\/\=]/g, ""); 
    while (i < input.length) { 
      enc1 = _keyStr.indexOf(input.charAt(i++)); 
      enc2 = _keyStr.indexOf(input.charAt(i++)); 
      enc3 = _keyStr.indexOf(input.charAt(i++)); 
      enc4 = _keyStr.indexOf(input.charAt(i++)); 
      chr1 = (enc1 << 2) | (enc2 >> 4); 
      chr2 = ((enc2 & 15) << 4) | (enc3 >> 2); 
      chr3 = ((enc3 & 3) << 6) | enc4; 
      output = output + String.fromCharCode(chr1); 
      if (enc3 != 64) { 
        output = output + String.fromCharCode(chr2); 
      } 
      if (enc4 != 64) { 
        output = output + String.fromCharCode(chr3); 
      } 
    } 
    output = _utf8_decode(output); 
    return output; 
  }  
  _utf8_decode = function (utftext) { 
    var string = ""; 
    var i = 0; 
    var c = c1 = c2 = 0; 
    while ( i < utftext.length ) { 
      c = utftext.charCodeAt(i); 
      if (c < 128) { 
        string += String.fromCharCode(c); 
        i++; 
      } else if((c > 191) && (c < 224)) { 
        c2 = utftext.charCodeAt(i+1); 
        string += String.fromCharCode(((c & 31) << 6) | (c2 & 63)); 
        i += 2; 
      } else { 
        c2 = utftext.charCodeAt(i+1); 
        c3 = utftext.charCodeAt(i+2); 
        string += String.fromCharCode(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63)); 
        i += 3; 
      } 
    } 
    return string; 
  } 
}
base_64_functions = new Base64();return base_64_functions.decode(str)
}

mongodb函数插入数据

db.system.js.insert({        _id : "base64_encode", 
      value : function base64(str){······ }
   });

创建场景

场景描述：

现有两种类型的设备分别为n1,n2，为保密传输（虽然base64一点都不安全，总比明文好点），需要对设备名称进行base64编码

初始化数据

db.test.insert({"name":"lisi",group:"n1"})
db.test.insert({"name":"zhangsan",group:"n1"})
db.test.insert({"name":"zhangsan1",group:"n2"})
db.test.insert({"name":"lisi1",group:"n2"})

创建mapreduce任务

创建map任务

var map=function () {emit(this.group, this.name); }

创建reduce任务（编码）

var reduce = function(key, values) {res = []; values.forEach(function(value){res.push({"key":key,"value":base64_encode(value)})}); res= {"res":res};return res}

执行任务

db.test.mapReduce(map,reduce,{query:{},out:"result"})
{
	"result" : "result",
	"timeMillis" : 31,
	"counts" : {
		"input" : 4,
		"emit" : 4,
		"reduce" : 2,
		"output" : 2
	},
	"ok" : 1,
	"operationTime" : Timestamp(1535969627, 5),
	"$clusterTime" : {
		"clusterTime" : Timestamp(1535969627, 5),
		"signature" : {
			"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
			"keyId" : NumberLong(0)
		}
	}
}

获取执行结果

db.result.find().pretty()
{
	"_id" : "n1",
	"value" : {
		"res" : [
			{
				"key" : "n1",
				"value" : "bGlzaQ=="
			},
			{
				"key" : "n1",
				"value" : "emhhbmdzYW4="
			}
		]
	}
}
{
	"_id" : "n2",
	"value" : {
		"res" : [
			{
				"key" : "n2",
				"value" : "emhhbmdzYW4x"
			},
			{
				"key" : "n2",
				"value" : "bGlzaTE="
			}
		]
	}
}

注意：map获取到的结果如果不是数组，则不会执行reduce，所以执行之前，与之后都需要去除没有执行reduce的数据

删除临时结果集

db.result.drop()

python中实现

代码如下：

#!/usr/bin/env python# -*- coding: utf-8 -*-""" @author: xingxiaolong @contact: [e-mail] @software: pdo @file: pymongo_mr.py @time: 2018/9/3 18:23 @desc: """from pymongo import MongoClientfrom bson.code import Codeif __name__ == "__main__":
    mongodb_client = MongoClient(host="IP", port=27017)
    test = mongodb_client.get_database(name="test")
    mapper = Code("""function () {emit(this.group, this.name); }""")
    reducer = Code("""function(key, values) {                        res = [];                        values.forEach(function(value){res.push({"key":key,"value":base64_encode(value)})});                        res= {"res":res};                        return res}""")
    result = test["test"].map_reduce(mapper, reducer, out="result",full_response=True)    for document in test["result"].find():        print document
    test["test"].drop()

执行结果：

{	"_id" : "n1",	"value" : {		"res" : [
			{				"key" : "n1",				"value" : "bGlzaQ=="
			},
			{				"key" : "n1",				"value" : "emhhbmdzYW4="
			}
		]
	}
}
{	"_id" : "n2",	"value" : {		"res" : [
			{				"key" : "n2",				"value" : "emhhbmdzYW4x"
			},
			{				"key" : "n2",				"value" : "bGlzaTE="
			}
		]
	}
}