clickhouse优化最佳实践(易企秀)

Posted

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了clickhouse优化最佳实践(易企秀)相关的知识，希望对你有一定的参考价值。

参考技术A Clickhouse堪称OLAP领域的黑马，最近发布的几个版本在多表关联分析上也有了极大的性能提升，尤其是还引入了Materializemysql Database Engine做到了实时对齐业务线mysql中的数据。

采样修饰符只有在mergetree engine表中才有效，且在创建表时需要指定采样策略；

clickhouse不支持设置多数据目录，为了提升数据io性能，可以挂载虚拟券组，一个券组绑定多块物理磁盘提升读写性能；多数查询场景SSD盘会比普通机械硬盘快2-3倍。

新版clickhouse提供了一个实验性的功能，那就是我们可以将clickhouse伪装成mysql的一个备库去实时对齐mysql中的数据，当mysql库表数据发生变化时会实时同步到clickhouse中；这样就省掉了单独维护实时spark/flink任务读取kafka数据再存入clickhouse的环节，大大降低了运维成本提升了效率。

为了避免因个别慢查询引起的服务雪崩问题，除了可以为单个查询设置超时以外，还可以配置周期熔断；在一个查询周期内，如果用户频繁进行慢查询操作超出规定阈值后将无法继续进行查询操作：

clickhouse权限管理与资源隔离
clickhouse高级功能上线之mysql实时数据同步

clickhouse如何构建复杂数据模型
clickhouse sql规范

易企秀H5 json配置文件解密分析

最近需要参考下易企秀H5的json配置文件，发现已经做了加密，其实前端的加密分析起来只是麻烦点。

抓包分析

先看一个H5： https://h5.eqxiu.com/s/XvEn30op

F12可以看到，配置json地址是：https://s1-cdn.eqxiu.com/eqs/page/142626394?code=XvEn30op&time=1542972816000

对应的json：

{"success":true,"code":200,"msg":"操作成功","obj":"加密后的字符串"}

obj对应的是正式的配置信息

解码分析

需要对加密信息进行解密，首先可以定位到解密代码

function _0x230bc7(_0x2fb175) {
    return _0x3c31('0xee') == typeof _0x2fb175[_0x3c31('0x25')] && _0x2fb175['x6fx62x6a'][_0x3c31('0xe')] > 0x64 ? _0x54c90c[_0x3c31('0x31f')]()['x74x68x65x6e'](function() {
        _0x249a60();
        var _0x5ab652 = null
          , _0x2cf0a4 = null
          , _0x4d1175 = null;
        try {
            var _0x3dbfaa = _0x2fb175[_0x3c31('0x25')]['x73x75x62x73x74x72x69x6ex67'](0x0, 0x13)
              , _0x360e25 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](0x13 + 0x10);
            _0x2cf0a4 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](0x13, 0x13 + 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x5ab652 = _0x3dbfaa + _0x360e25,
            _0x2cf0a4 = CryptoJS['x65x6ex63'][_0x3c31('0x320')][_0x3c31('0x6b')](_0x2cf0a4),
            _0x4d1175 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')]['x70x61x72x73x65'](_0x4d1175);
            var _0x57e61a = CryptoJS[_0x3c31('0x322')][_0x3c31('0x323')](_0x5ab652, _0x2cf0a4, {
                'x69x76': _0x4d1175,
                'x6dx6fx64x65': CryptoJS[_0x3c31('0x324')][_0x3c31('0x325')],
                'x70x61x64x64x69x6ex67': CryptoJS[_0x3c31('0x326')][_0x3c31('0x327')]
            });
            return _0x2fb175[_0x3c31('0x36')] = JSON[_0x3c31('0x6b')](CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x267')](_0x57e61a)),
            _0x2fb175;
        } catch (_0x36fffe) {
            _0x5ab652 = _0x2fb175[_0x3c31('0x25')]['x73x75x62x73x74x72x69x6ex67'](0x0, _0x2fb175[_0x3c31('0x25')][_0x3c31('0xe')] - 0x10),
            _0x2cf0a4 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](_0x2fb175[_0x3c31('0x25')][_0x3c31('0xe')] - 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x2cf0a4 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x6b')](_0x2cf0a4),
            _0x4d1175 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x6b')](_0x4d1175);
            var _0x4ff7de = CryptoJS[_0x3c31('0x322')][_0x3c31('0x323')](_0x5ab652, _0x2cf0a4, {
                'x69x76': _0x4d1175,
                'x6dx6fx64x65': CryptoJS['x6dx6fx64x65'][_0x3c31('0x325')],
                'x70x61x64x64x69x6ex67': CryptoJS[_0x3c31('0x326')][_0x3c31('0x327')]
            });
            return _0x2fb175[_0x3c31('0x36')] = JSON[_0x3c31('0x6b')](CryptoJS[_0x3c31('0x321')]['x55x74x66x38'][_0x3c31('0x267')](_0x4ff7de)),
            _0x2fb175;
        }
    }) : Promise[_0x3c31('0x1e')](_0x2fb175);

这个代码基本不可读，简单分析下可以发现，_0x3c31(‘0x321‘)对应一个字符串，‘x6fx62x6a‘等也可以转义：

先转义：

function decode(xData) {
    return xData.replace(/\x(w{2})/g, function (_, $1) { return String.fromCharCode(parseInt($1, 16)) });
}

然后替换下：

Function.prototype.getMultiLine = function () {
    var lines = new String(this);
    lines = lines.substring(lines.indexOf("/*") + 3, lines.lastIndexOf("*/"));
    return lines;
}

function decode(xData) {
    return xData.replace(/\x(w{2})/g, function (_, $1) { return String.fromCharCode(parseInt($1, 16)) });
}

var str1 = function () {
    /* 
   var _0x5ab652 = _0x50019d(_0x3c31('0x30c'))
, _0x2cf0a4 = _0x50019d('x63x6fx6dx70x4bx65x79')
, _0x5cba8a = {
  'x74x79x70x65': _0x3c31('0x16c'),
  'x75x72x6c': _0x8aa6f1()
}
, _0xfca4af = {
  'x74x79x70x65': _0x3c31('0x16c'),
  'x75x72x6c': _0xefaeb()
};
_0x31f1b8 && (_0x5cba8a[_0x3c31('0x2cb')] = {
  'x70x61x73x73x77x6fx72x64': _0x31f1b8
});
var _0x11871b = null
, _0x170c7e = Promise[_0x3c31('0x1e')](null);
 
var _0x256cd0 = _0x2cf0a4(0x16)
, _0x36150e = _0x2cf0a4(0x15)
, _0x42a8d9 = _0x36150e['x61x6ax61x78']
, _0xdc46dc = _0x36150e[_0x3c31('0x1ef')]
, _0xcca797 = _0x2cf0a4(0x18)
, _0x50019d = _0xcca797[_0x3c31('0x5f')]
, _0x5c77e3 = _0xcca797['x70x61x72x73x65x55x72x6c']
, _0x36d54a = _0x2cf0a4(0x3a)['x70x65x72x66x65x63x74x4dx65x74x61']
, _0x4c6a9e = _0x2cf0a4(0x2c)[_0x3c31('0x328')]
, _0x296a9b = _0x2cf0a4(0x2c)[_0x3c31('0x329')]
, _0x54c90c = _0x2cf0a4(0x17)
, _0x50f238 = _0x2cf0a4(0x3d)[_0x3c31('0x32a')]
, 0x13 = 0x13
, 0x0 = 0x0
, 0x10 = 0x10
, CryptoJS = null;
    function _0x230bc7(_0x2fb175) {
    return _0x3c31('0xee') == typeof _0x2fb175[_0x3c31('0x25')] && _0x2fb175['x6fx62x6a'][_0x3c31('0xe')] > 0x64 ? _0x54c90c[_0x3c31('0x31f')]()['x74x68x65x6e'](function() {
        _0x249a60();
        var _0x5ab652 = null
          , _0x2cf0a4 = null
          , _0x4d1175 = null;
        try {
            var _0x3dbfaa = _0x2fb175[_0x3c31('0x25')]['x73x75x62x73x74x72x69x6ex67'](0x0, 0x13)
              , _0x360e25 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](0x13 + 0x10);
            _0x2cf0a4 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](0x13, 0x13 + 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x5ab652 = _0x3dbfaa + _0x360e25,
            _0x2cf0a4 = CryptoJS['x65x6ex63'][_0x3c31('0x320')][_0x3c31('0x6b')](_0x2cf0a4),
            _0x4d1175 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')]['x70x61x72x73x65'](_0x4d1175);
            var _0x57e61a = CryptoJS[_0x3c31('0x322')][_0x3c31('0x323')](_0x5ab652, _0x2cf0a4, {
                'x69x76': _0x4d1175,
                'x6dx6fx64x65': CryptoJS[_0x3c31('0x324')][_0x3c31('0x325')],
                'x70x61x64x64x69x6ex67': CryptoJS[_0x3c31('0x326')][_0x3c31('0x327')]
            });
            return _0x2fb175[_0x3c31('0x36')] = JSON[_0x3c31('0x6b')](CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x267')](_0x57e61a)),
            _0x2fb175;
        } catch (_0x36fffe) {
            _0x5ab652 = _0x2fb175[_0x3c31('0x25')]['x73x75x62x73x74x72x69x6ex67'](0x0, _0x2fb175[_0x3c31('0x25')][_0x3c31('0xe')] - 0x10),
            _0x2cf0a4 = _0x2fb175[_0x3c31('0x25')][_0x3c31('0xeb')](_0x2fb175[_0x3c31('0x25')][_0x3c31('0xe')] - 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x2cf0a4 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x6b')](_0x2cf0a4),
            _0x4d1175 = CryptoJS[_0x3c31('0x321')][_0x3c31('0x320')][_0x3c31('0x6b')](_0x4d1175);
            var _0x4ff7de = CryptoJS[_0x3c31('0x322')][_0x3c31('0x323')](_0x5ab652, _0x2cf0a4, {
                'x69x76': _0x4d1175,
                'x6dx6fx64x65': CryptoJS['x6dx6fx64x65'][_0x3c31('0x325')],
                'x70x61x64x64x69x6ex67': CryptoJS[_0x3c31('0x326')][_0x3c31('0x327')]
            });
            return _0x2fb175[_0x3c31('0x36')] = JSON[_0x3c31('0x6b')](CryptoJS[_0x3c31('0x321')]['x55x74x66x38'][_0x3c31('0x267')](_0x4ff7de)),
            _0x2fb175;
        }
    }) : Promise[_0x3c31('0x1e')](_0x2fb175);
}"
  */
}
var js1 = decode(str1.getMultiLine());
js1 = js1.replace(/_0x3c31('([^']+)')/g, function ($v, $g) { return _0x3c31($g); })

得到

var _0x5ab652 = _0x50019d(userKey)
, _0x2cf0a4 = _0x50019d('compKey')
, _0x5cba8a = {
  'type': GET,
  'url': _0x8aa6f1()
}
, _0xfca4af = {
  'type': GET,
  'url': _0xefaeb()
};
_0x31f1b8 && (_0x5cba8a[data] = {
  'password': _0x31f1b8
});
var _0x11871b = null
, _0x170c7e = Promise[resolve](null);
 
var _0x256cd0 = _0x2cf0a4(0x16)
, _0x36150e = _0x2cf0a4(0x15)
, _0x42a8d9 = _0x36150e['ajax']
, _0xdc46dc = _0x36150e[$ajax]
, _0xcca797 = _0x2cf0a4(0x18)
, _0x50019d = _0xcca797[getUrlParam]
, _0x5c77e3 = _0xcca797['parseUrl']
, _0x36d54a = _0x2cf0a4(0x3a)['perfectMeta']
, _0x4c6a9e = _0x2cf0a4(0x2c)[isVipScene]
, _0x296a9b = _0x2cf0a4(0x2c)[isTgScene]
, _0x54c90c = _0x2cf0a4(0x17)
, _0x50f238 = _0x2cf0a4(0x3d)[setJsCrypto]
, 0x13 = 0x13
, 0x0 = 0x0
, 0x10 = 0x10
, CryptoJS = null;
    function _0x230bc7(_0x2fb175) {
    return string == typeof _0x2fb175[obj] && _0x2fb175['obj'][length] > 0x64 ? _0x54c90c[$loadCryptoJS]()['then'](function() {
        _0x249a60();
        var _0x5ab652 = null
          , _0x2cf0a4 = null
          , _0x4d1175 = null;
        try {
            var _0x3dbfaa = _0x2fb175[obj]['substring'](0x0, 0x13)
              , _0x360e25 = _0x2fb175[obj][substring](0x13 + 0x10);
            _0x2cf0a4 = _0x2fb175[obj][substring](0x13, 0x13 + 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x5ab652 = _0x3dbfaa + _0x360e25,
            _0x2cf0a4 = CryptoJS['enc'][Utf8][parse](_0x2cf0a4),
            _0x4d1175 = CryptoJS[enc][Utf8]['parse'](_0x4d1175);
            var _0x57e61a = CryptoJS[AES][decrypt](_0x5ab652, _0x2cf0a4, {
                'iv': _0x4d1175,
                'mode': CryptoJS[mode][CFB],
                'padding': CryptoJS[pad][NoPadding]
            });
            return _0x2fb175[list] = JSON[parse](CryptoJS[enc][Utf8][stringify](_0x57e61a)),
            _0x2fb175;
        } catch (_0x36fffe) {
            _0x5ab652 = _0x2fb175[obj]['substring'](0x0, _0x2fb175[obj][length] - 0x10),
            _0x2cf0a4 = _0x2fb175[obj][substring](_0x2fb175[obj][length] - 0x10),
            _0x4d1175 = _0x2cf0a4,
            _0x2cf0a4 = CryptoJS[enc][Utf8][parse](_0x2cf0a4),
            _0x4d1175 = CryptoJS[enc][Utf8][parse](_0x4d1175);
            var _0x4ff7de = CryptoJS[AES][decrypt](_0x5ab652, _0x2cf0a4, {
                'iv': _0x4d1175,
                'mode': CryptoJS['mode'][CFB],
                'padding': CryptoJS[pad][NoPadding]
            });
            return _0x2fb175[list] = JSON[parse](CryptoJS[enc]['Utf8'][stringify](_0x4ff7de)),
            _0x2fb175;
        }
    }) : Promise[resolve](_0x2fb175);
}"

基板上可以读了，使用CryptoJS做的前端解密，然后直接给最后的代码：


// 依赖： https://lib.eqh5.com/CryptoJS/1.0.1/cryptoJs.js
function decrypt(result) {
    var ciphertext = null
        , key = null
        , iv = null;
    try {
        var part0 = result.obj.substring(0x0, 0x13)
            , part1 = result.obj.substring(0x13 + 0x10);
        key = result.obj.substring(0x13, 0x13 + 0x10),
            iv = key,
            ciphertext = part0 + part1,
            key = CryptoJS.enc.Utf8.parse(key),
            iv = CryptoJS.enc.Utf8.parse(iv);
        var decryptData = CryptoJS.AES.decrypt(ciphertext, key, {
            'iv': iv,
            'mode': CryptoJS.mode.CFB,
            'padding': CryptoJS.pad.NoPadding
        });
        return CryptoJS.enc.Utf8.stringify(decryptData);
    } catch (_0x36fffe) {
        ciphertext = result[obj]['substring'](0x0, result.obj.length - 0x10),
            key = result[obj][substring](result.obj.length - 0x10),
            iv = key,
            key = CryptoJS.enc.Utf8.parse(key),
            iv = CryptoJS.enc.Utf8.parse(iv);
        var decryptData = CryptoJS.AES.decrypt(ciphertext, key, {
            'iv': iv,
            'mode': CryptoJS.mode.CFB,
            'padding': CryptoJS.pad.NoPadding
        });
        return CryptoJS.enc.Utf8.stringify(decryptData)
    }
}

以上是关于clickhouse优化最佳实践(易企秀)的主要内容，如果未能解决你的问题，请参考以下文章

基于Flink+ClickHouse构建实时游戏数据分析最佳实践

最佳实践携程ClickHouse日志分析实践

最佳实践携程ClickHouse日志分析实践

ClickHouse集群部署 全网最佳实践

最佳实践ClickHouse在携程酒店数仓的实践

Flink+Clickhouse实时数仓在广投集团的最佳实践