MongoShake数据同步工具

Posted 2020-12-13

1、下载

wget https://github.com/alibaba/MongoShake/releases/download/release-v2.4.6-20200617/mongo-shake-v2.4.6.tar.gz

2.安装

 tar -zxvf mongo-shake-v2.4.6.tar.gz 
mongo-shake-v2.4.6/
mongo-shake-v2.4.6/receiver.linux
mongo-shake-v2.4.6/collector.windows
mongo-shake-v2.4.6/mongoshake-stat
mongo-shake-v2.4.6/ChangeLog
mongo-shake-v2.4.6/collector.linux
mongo-shake-v2.4.6/stop.sh
mongo-shake-v2.4.6/start.sh
mongo-shake-v2.4.6/comparison.py
mongo-shake-v2.4.6/receiver.windows
mongo-shake-v2.4.6/receiver.darwin
mongo-shake-v2.4.6/hypervisor
mongo-shake-v2.4.6/collector.conf
mongo-shake-v2.4.6/collector.darwin

cd mongo-shake-v2.4.6

3.配置

#同步模式为所有同步
sync_mode = all
#源mongodb
mongo_urls = mongodb://root:rootpa***@192.168.1.28:30001,192.168.1.29:30001,192.168.1.27:30001
#源mongodb的config
mongo_cs_url = mongodb://root:rootpa***@192.168.1.28:30002,192.168.1.29:30002,192.168.1.27:30002
#源mongodb的mongos
mongo_s_url = mongodb://root:rootpa***@192.168.1.28:40001 
tunnel = direct
#目标mongodb
tunnel.address = mongodb://root:rootpa***@192.168.1.12:30001
tunnel.message = raw
mongo_connect_mode = secondaryPreferred
#要迁移的db
filter.namespace.white = test
#开启DDL同步
filter.ddl_enable = true
checkpoint.storage.url = mongodb://root:rootpa***@192.168.1.12:30001

4.启动

./collector.linux -conf=collector.conf

5.日志

tail -f logs/collector.log 
[2020/06/28 17:35:09 CST] [INFO] [collector.(*Worker).transfer:189] Collector-worker-3 transfer retransmit:false send [1] logs. reply_acked [6843329907074793473[1593336907, 1]], list_unack [0] 
[2020/06/28 17:35:09 CST] [INFO] [common.(*ReplicationMetric).startup.func1:175] [name=shard1, stage=incr, get=307153, filter=306137, success=998, tps=2, ckpt_times=299, lsn_ckpt={6843329829765382145[1593336889, 1], 2020-06-28 17:34:49}, lsn_ack={6843329907074793473[1593336907, 1], 2020-06-28 17:35:07}]]
[2020/06/28 17:35:12 CST] [INFO] [collector.(*OplogSyncer).calculateWorkerLowestCheckpoint:189] worker offset [6843329877010022401 6843329907074793473] use lowest 6843329877010022401[1593336900, 1]
[2020/06/28 17:35:12 CST] [INFO] [executor.(*Executor).doSync:243] Replayer-3 Executor-3 doSync oplogRecords received[1] merged[1]. merge to 100.00% chunks
[2020/06/28 17:35:12 CST] [INFO] [collector.(*Worker).transfer:189] Collector-worker-3 transfer retransmit:false send [1] logs. reply_acked [6843329924254662657[1593336911, 1]], list_unack [0] 
[2020/06/28 17:35:12 CST] [INFO] [ckpt.(*MongoCheckpoint).Insert:134] shard1 Record new checkpoint success [1593336900]
[2020/06/28 17:35:12 CST] [INFO] [collector.(*OplogSyncer).checkpoint:121] CheckpointOperation write success. updated from 6843329829765382145[1593336889, 1] to 6843329877010022401[1593336900, 1]
[2020/06/28 17:35:12 CST] [INFO] [executor.(*Executor).doSync:243] Replayer-2 Executor-2 doSync oplogRecords received[1] merged[1]. merge to 100.00% chunks
[2020/06/28 17:35:12 CST] [INFO] [collector.(*Worker).transfer:189] Collector-worker-2 transfer retransmit:false send [1] logs. reply_acked [6843329919959695379[1593336910, 19]], list_unack [0] 
[2020/06/28 17:35:14 CST] [INFO] [common.(*ReplicationMetric).startup.func1:175] [name=shard1, stage=incr, get=307392, filter=306286, success=1000, tps=0, ckpt_times=300, lsn_ckpt={6843329877010022401[1593336900, 1], 2020-06-28 17:35:00}, lsn_ack={6843329924254662657[1593336911, 1], 2020-06-28 17:35:11}]]