SpringBoot迁移数据库数据到ES+操纵ES进行高级检索源码

Posted 2021-03-12 官萧何

实现类源码

package com.txj.bwbd.es.esService.impl;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.bean.copier.CopyOptions;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.txj.bwbd.common.CommonConstraint;
import com.txj.bwbd.es.dto.SearchDto;
import com.txj.bwbd.es.esDao.BwbdTypeDao;
import com.txj.bwbd.es.esDao.FgTypeDao;
import com.txj.bwbd.es.esEntity.BwbdType;
import com.txj.bwbd.es.esService.BwbdTypeService;
import com.txj.bwbd.es.esService.FgTypeService;
import com.txj.bwbd.sqlserver.entity.AlTestR3;
import com.txj.bwbd.sqlserver.entity.FgTestR3;
import com.txj.bwbd.sqlserver.entity.WdTestR3;
import com.txj.bwbd.sqlserver.service.IAlTestR3Service;
import com.txj.bwbd.sqlserver.service.IFgTestR3Service;
import com.txj.bwbd.sqlserver.service.IWdTestR3Service;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;


/**
 * @author: 官昌洪
 * @Description:
 * @Date: 2020/2/1 10:31
 * @Param:
 * @return:
 */
@Service
public class BwbdTypeServiceImpl implements BwbdTypeService {

    @Autowired
    BwbdTypeDao bwbdTypeDao;

    @Autowired
    RestHighLevelClient client;

    @Autowired
    IFgTestR3Service iFgTestR3Service;

    @Autowired
    IWdTestR3Service iWdTestR3Service;

    @Autowired
    IAlTestR3Service iAlTestR3Service;

    @Override
    @Transactional(rollbackFor = Exception.class)
    public void syncFg() {
        List<FgTestR3> r3s = iFgTestR3Service.list((new QueryWrapper<FgTestR3>())
                .select("webGuid").groupBy("webGuid"));
        List<String> webGuids = r3s.stream().filter(fgTestR3 -> StringUtils.isNotEmpty(fgTestR3
                .getWebGuid())).map(fgTestR3 -> fgTestR3.getWebGuid()).collect(Collectors.toList());

        for (String webGuid : webGuids) {
            List<FgTestR3> fgTestR3s = iFgTestR3Service.list((new QueryWrapper<FgTestR3>())
                    .eq("webGuid", webGuid));
            List<BwbdType> bwbdTypes = new ArrayList<>();
            for (FgTestR3 fgTestR3 : fgTestR3s) {
                BwbdType bwbdType = new BwbdType();
                BeanUtil.copyProperties(fgTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
                bwbdType.setId(BwbdType.DATA_TYPE_FG + BwbdType.ID_SPLIT + fgTestR3.getAutoid());
                bwbdType.setDataType(BwbdType.DATA_TYPE_FG);
                bwbdTypes.add(bwbdType);
            }
            bwbdTypeDao.saveAll(bwbdTypes);
        }

    }

    @Override
    @Transactional(rollbackFor = Exception.class)
    public void syncWd() {
        List<WdTestR3> r3s = iWdTestR3Service.list((new QueryWrapper<WdTestR3>())
                .select("webGuid").groupBy("webGuid"));
        List<String> webGuids = r3s.stream().filter(wdTestR3 -> StringUtils.isNotEmpty(wdTestR3
                .getWebGuid())).map(wdTestR3 -> wdTestR3.getWebGuid()).collect(Collectors.toList());

        for (String webGuid : webGuids) {
            List<WdTestR3> wdTestR3s = iWdTestR3Service.list((new QueryWrapper<WdTestR3>())
                    .eq("webGuid", webGuid));
            List<BwbdType> bwbdTypes = new ArrayList<>();
            for (WdTestR3 wdTestR3 : wdTestR3s) {
                BwbdType bwbdType = new BwbdType();
                BeanUtil.copyProperties(wdTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
                bwbdType.setContents(wdTestR3.getRequestContent() + wdTestR3.getContents()
                        + wdTestR3.getResponseContent());
                bwbdType.setId(BwbdType.DATA_TYPE_WD + BwbdType.ID_SPLIT + wdTestR3.getAutoid());
                bwbdType.setDataType(BwbdType.DATA_TYPE_WD);
                bwbdTypes.add(bwbdType);
            }
            bwbdTypeDao.saveAll(bwbdTypes);
        }

    }

    @Override
    @Transactional(rollbackFor = Exception.class)
    public void syncAl() {
        List<AlTestR3> r3s = iAlTestR3Service.list((new QueryWrapper<AlTestR3>())
                .select("webGuid").groupBy("webGuid"));
        List<String> webGuids = r3s.stream().filter(fgTestR3 -> StringUtils.isNotEmpty(fgTestR3
                .getWebGuid())).map(fgTestR3 -> fgTestR3.getWebGuid()).collect(Collectors.toList());

        for (String webGuid : webGuids) {
            List<AlTestR3> alTestR3s = iAlTestR3Service.list((new QueryWrapper<AlTestR3>())
                    .eq("webGuid", webGuid));
            List<BwbdType> bwbdTypes = new ArrayList<>();
            for (AlTestR3 alTestR3 : alTestR3s) {
                BwbdType bwbdType = new BwbdType();
                BeanUtil.copyProperties(alTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
                bwbdType.setId(BwbdType.DATA_TYPE_AL + BwbdType.ID_SPLIT + alTestR3.getAutoid());
                bwbdType.setDataType(BwbdType.DATA_TYPE_AL);
                bwbdType.setNumbers(alTestR3.getCaseNumber());
                bwbdTypes.add(bwbdType);
            }

            if (bwbdTypes.size() > 20000) {
                splitSave(bwbdTypes);
            } else {
                bwbdTypeDao.saveAll(bwbdTypes);
            } 
        }

    }

    /**
     * @author: 官昌洪
     * @Description: 直接保存会内存溢出，切片保存数据
     * @Date: 2020/2/5 11:43
     * @Param: 
     * @return: 
     */
    private void splitSave(List<BwbdType> bwbdTypes) {
        int size = bwbdTypes.size();
        int step = 2000;
        int fromIndex = 0;
        int toIndex = step;
        while (toIndex < size) {
            List<BwbdType> optList = bwbdTypes.subList(fromIndex, toIndex);
            bwbdTypeDao.saveAll(optList);
            fromIndex = toIndex;
            toIndex = toIndex + step;
        }

        if (fromIndex < size) {
            toIndex = size;
            List<BwbdType> optList = bwbdTypes.subList(fromIndex, toIndex);
            bwbdTypeDao.saveAll(optList);
        }
    }

    @Override
    public List<BwbdType> improveSearch(SearchDto searchDto) {

        String text = searchDto.getTerm();
        text = QueryParser.escape(text);  // 主要就是这一句把特殊字符都转义,那么lucene就可以识别
        // 搜索请求对象
        SearchRequest searchRequest = new SearchRequest(BwbdType.ES_INDEX);
        // 指定类型
        searchRequest.types(BwbdType.ES_TYPE);
        // 搜索源构建对象
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        // 搜索方式
        // 首先构造多关键字查询条件
        MultiMatchQueryBuilder matchQueryBuilder = QueryBuilders
                .multiMatchQuery(text, BwbdType.PROPERTY_NUMBERS
                        , BwbdType.PROPERTY_TITLES, BwbdType.PROPERTY_CONTENTS)
                .field(BwbdType.PROPERTY_NUMBERS, 100)
                .field(BwbdType.PROPERTY_TITLES, 10)
                .field(BwbdType.PROPERTY_CONTENTS, 1).minimumShouldMatch(BwbdType.MATCH_LEVEL_THREE);
        // 添加条件到布尔查询
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(matchQueryBuilder);
//        // 通过布尔查询来构造过滤查询
//        boolQueryBuilder.filter(QueryBuilders.matchQuery("economics","L"));
        // 将查询条件封装给查询对象
        searchSourceBuilder.query(boolQueryBuilder);
        searchSourceBuilder.size(searchDto.getSize());
        searchSourceBuilder.from(searchDto.getPage() - 1);
        // ***********************

        // 高亮查询
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.preTags(CommonConstraint.LIGHT_TAG_START); // 高亮前缀
        highlightBuilder.postTags(CommonConstraint.LIGHT_TAG_END); // 高亮后缀
        List<HighlightBuilder.Field> fields = highlightBuilder.fields();
        fields.add(new HighlightBuilder
                .Field(BwbdType.PROPERTY_NUMBERS)); // 高亮字段
        fields.add(new HighlightBuilder
                .Field(BwbdType.PROPERTY_TITLES)); // 高亮字段
        fields.add(new HighlightBuilder
                .Field(BwbdType.PROPERTY_CONTENTS)); // 高亮字段
        // 添加高亮查询条件到搜索源
        searchSourceBuilder.highlighter(highlightBuilder);

        // ***********************

//        // 设置源字段过虑,第一个参数结果集包括哪些字段，第二个参数表示结果集不包括哪些字段
//        searchSourceBuilder.fetchSource(new String[]{"name","studymodel","price","timestamp"},new String[]{});
        // 向搜索请求对象中设置搜索源
        searchRequest.source(searchSourceBuilder);
        // 执行搜索,向ES发起http请求
        SearchResponse searchResponse = null;
        try {
            searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }
        List<BwbdType> bwbdTypes = obtainFgType(searchResponse);


        return bwbdTypes;
    }

    private List<BwbdType> obtainFgType(SearchResponse searchResponse) {
        // 搜索结果
        SearchHits hits = searchResponse.getHits();
        // 匹配到的总记录数
        long totalHits = hits.getTotalHits();
        // 得到匹配度高的文档
        SearchHit[] searchHits = hits.getHits();

        List<BwbdType> bwbdTypes = new ArrayList<>();

        for (SearchHit hit : searchHits) {
            String content = hit.getSourceAsString();//使用ES的java接口将实体类对应的内容转换为json字符串
            BwbdType bwbdType = JSONObject.parseObject(content,BwbdType.class); //生成pojo对象
            // 获取高亮查询的内容。如果存在，则替换原来的name
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            if( highlightFields != null ){
                HighlightField nameField = highlightFields.get(bwbdType.PROPERTY_NUMBERS);
                if(nameField!=null){
                    Text[] fragments = nameField.getFragments();
                    StringBuffer stringBuffer = new StringBuffer();
                    for (Text str : fragments) {
                        stringBuffer.append(str.string());
                    }
                    String numbers = stringBuffer.toString();
                    bwbdType.setNumbers(numbers);
                }

                HighlightField titlesField = highlightFields.get(bwbdType.PROPERTY_TITLES);
                if(titlesField!=null){
                    Text[] fragments = titlesField.getFragments();
                    StringBuffer stringBuffer = new StringBuffer();
                    for (Text str : fragments) {
                        stringBuffer.append(str.string());
                    }
                    String titles = stringBuffer.toString();
                    bwbdType.setTitles(titles);
                }

                HighlightField contentsField = highlightFields.get(bwbdType.PROPERTY_CONTENTS);
                if(contentsField!=null){
                    Text[] fragments = contentsField.getFragments();
                    StringBuffer stringBuffer = new StringBuffer();
                    for (Text str : fragments) {
                        stringBuffer.append(str.string());
                    }
                    String contents = stringBuffer.toString();
                    bwbdType.setContents(contents);
                }
            }
            bwbdTypes.add(bwbdType);
        }
        return bwbdTypes;
    }
}