elasticsearch 数据迁移

Posted 2020-11-10

现需要将某集群下一个索引下的所有数据迁移到另一个集群上， elasticsearch-dump， Elasticsearch-Exporter 试了一下都不好使，只能老实的写代码来实现

import os
import sys
import pyes
import datetime

index_list = [
    ["alias-offer", "offer"]
]

ES_URL = "http://ip1:9200/"
NEW_ES_URL = "http://ip2:9200/"

def main():
    for _index, _type in index_list:
        print "开始索引:"+_index
        conn = pyes.es.ES(ES_URL)
        print "已经连接上原地址"
        search = pyes.query.MatchAllQuery().search(bulk_read=1000)
        hits = conn.search(search, _index, _type, scan=True, scroll="30m", model=lambda _,hit: hit)
        print "原地址返回数据"
        conn2 = pyes.es.ES(NEW_ES_URL)
        print "连接到新地址"
        count = 0
        for hit in hits:
            conn2.index(hit[‘_source‘], _index, _type, hit[‘_id‘], bulk=True)
            count += 1
            if count % 1000 == 0:
                print (datetime.datetime.now().strftime(‘%Y-%m-%d %H:%M:%S‘),count)
                conn2.flush_bulk(forced=True)
        conn2.flush_bulk(forced=True)
        conn2 = None
        conn = None

if __name__ == ‘__main__‘:
    main()
    print "数据导入结束"

虽然慢，但能用，基本满足需要！