springboot引入Elasticsearch

Posted 2023-03-11 梦想载旺

之前在工程中日志信息存储在mysql中，由于日志越来越大，导致查询效率越来越慢，想着运用Elasticsearch提高效率，自己结合情况，花了半天时间写了一套springboot引用Elasticsearch。

一 ：创建SpringBoot项目并引入Elasticsearch依赖

引入相关jar包，具体如下（注意与springboot版本相对应）：

        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>7.17.9</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>7.17.9</version>
        </dependency>

        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>7.17.9</version>
        </dependency>

二 ：SpringBoot项目配置相应配置信息

配置相应的Elasticsearch，具体如下：

spring:
  elasticsearch:
    host: 127.0.0.1
    port: 9200
    connTimeout: 3000
    socketTimeout: 5000
    connectionRequestTimeout: 500

三 ：SpringBoot项目配置相应配置客户端

配置读取配置的类：

import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

@Data
@Component
public class EsConfig 

    @Value("$spring.elasticsearch.host")
    private String host;

    @Value("$spring.elasticsearch.port")
    private int port;

    @Value("$spring.elasticsearch.connTimeout")
    private int connTimeout;

    @Value("$spring.elasticsearch.socketTimeout")
    private int socketTimeout;

    @Value("$spring.elasticsearch.connectionRequestTimeout")
    private int connectionRequestTimeout;

创建连接Elasticsearch连接的客户端：

import com.huanggr.common.core.domain.EsConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;

@Slf4j
@Configuration
@Component
public class ElasticsearchConfiguration 
    @Autowired
    EsConfig esConfig;


    @Bean(destroyMethod = "close", name = "client")
    public RestHighLevelClient initRestClient() 
        RestClientBuilder builder = RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort()))
                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
                        .setConnectTimeout(esConfig.getConnTimeout())
                        .setSocketTimeout(esConfig.getSocketTimeout())
                        .setConnectionRequestTimeout(esConfig.getConnectionRequestTimeout()));
        return new RestHighLevelClient(builder);
    

    // 注册 rest高级客户端
    @Bean
    public RestHighLevelClient restHighLevelClient() 
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost(esConfig.getHost(), esConfig.getPort(), "http")
                )
        );
        return client;
    

四 ：配置相应的Elasticsearch的操作类

配置相应的枚举：

/**
 * Es查询类型
 *
 * @author huanggr
 */

public enum EsQueryType

    term("term", "精确值查找"),
    range("range", "范围检索"),
    prefix("prefix", "模糊查询");

    public  String code;
    public  String value;

    EsQueryType(String code, String value)
    
        this.code = code;
        this.value = value;
    

    public String getCode()
    
        return code;
    

    public String getValue()
    
        return value;
    

public enum RangeType 

    lte("lte","小于或等于"),
    lt("lt","小于"),
    gt("gt","大于"),
    gte("gte","大于或等于");

    private final String code;
    private final String value;

    RangeType(String code, String value)
    
        this.code = code;
        this.value = value;
    

    public String getCode()
    
        return code;
    

    public String getValue()
    
        return value;
    

配置相应的实体类：

import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

@Data
public class EsQueryVo 
    @ApiModelProperty(value = "参数主键")
    private String name;

    @ApiModelProperty(value = "参数主键值")
    private String value;

    @ApiModelProperty(value = "查询类型")
    private String queryType;

    @ApiModelProperty(value = "大于或者大于等于或者小于或者等于")
    private String gteandlte;

    public EsQueryVo()

    
    public EsQueryVo(String name, String value,String queryType,String gteandlte)
        this.name = name;
        this.value = value;
        this.queryType = queryType;
        this.gteandlte = gteandlte;
    

import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.Map;

@Data
public class PageList 
    @ApiModelProperty(value = "集合")
    private List<Map<String,Object>> hitList;

    @ApiModelProperty(value = "总数")
    private long total;

配置EsManageService相应操作管理类：

import com.alibaba.fastjson2.JSON;
import com.huanggr.common.core.domain.EsQueryVo;
import com.huanggr.common.core.domain.PageList;
import com.huanggr.common.core.enums.EsQueryType;
import com.huanggr.common.core.enums.RangeType;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@Slf4j
public class EsManageService 

    @Autowired
    @Qualifier("restHighLevelClient")
    public RestHighLevelClient client;

    /**
     * 创建索引
     * @param index 索引名称
     * @param map   key的Map
     * @return boolean
     * @throws IOException
     */
    public boolean createIndex(String index,Map<String,Object> map) throws IOException 
        //校验索引是否存在
        GetIndexRequest getRequest = new GetIndexRequest(index);
        if(client.indices().exists(getRequest,RequestOptions.DEFAULT))
            log.info("已存在索引",index);
            return true;
        
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
        createIndexRequest.settings(Settings.builder()
                .put("index.number_of_shards", 1)
                .put("index.number_of_replicas", 0)
        );
        createIndexRequest.mapping("_doc",map);
        CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
        return  createIndexResponse.isAcknowledged();
    

    /**
     * 创建索引
     * @param index 索引名称
     * @param JsonString   String
     * @return boolean
     * @throws IOException
     */
    public boolean createIndex(String index,String JsonString)throws IOException 
        //校验索引是否存在
        GetIndexRequest getRequest = new GetIndexRequest(index);
        if(client.indices().exists(getRequest,RequestOptions.DEFAULT))
            log.info("已存在索引",index);
            return true;
        
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
        createIndexRequest.settings(Settings.builder()
                .put("index.number_of_shards", 1)
                .put("index.number_of_replicas", 0)
        );
        createIndexRequest.mapping("_doc",JsonString, XContentType.JSON);
        CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
        return  createIndexResponse.isAcknowledged();
    


    //删除索引(删表)
    public Boolean deleteIndex(String index) throws IOException 
        DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index);
        AcknowledgedResponse deleteIndexResponse = client.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
        return deleteIndexResponse.isAcknowledged();
    



    //创建文档(插入数据)
    public Boolean createDocumentByHashMap(HashMap<String,Object> map, String index, String id) throws Exception 

        IndexRequest indexRequest = new IndexRequest(index)
                .id(id).source(map);
        IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
        return ((IndexResponse) indexResponse).status().equals(RestStatus.OK);
    


    //更新文档
    public Boolean updateDocumentByMap(HashMap<String,Object> map, String index, String id) throws Exception 
        UpdateRequest updateRequest = new UpdateRequest(index, id);
        updateRequest.doc(map);
        UpdateResponse updateResponse = client.update(updateRequest, RequestOptions.DEFAULT);
        return updateResponse.status().equals(RestStatus.OK);
    


    //删除文档
    public String deleteDocument(String id,String index) throws Exception 
        DeleteRequest deleteRequest = new DeleteRequest(index, id);
        DeleteResponse response = client.delete(deleteRequest, RequestOptions.DEFAULT);
        return response.getResult().name();
    

    //查询Es文档
    public PageList searchListByPage(String index, List<EsQueryVo> esQueryVos, int pageNum, int pageSize) throws IOException 
        PageList pageList = new PageList();
        List<Map<String,Object>> hitList = new ArrayList();
        SearchRequest searchRequest = new SearchRequest(index);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
        if(esQueryVos != null && esQueryVos.size()>0)
            for (EsQueryVo esQueryVo:
                    esQueryVos) 
                clboolQueryBuilder(queryBuilder,esQueryVo);
            
        else
            queryBuilder.must(QueryBuilders.matchAllQuery());
        
        log.info("queryBuilder查询条件:", queryBuilder.toString());
        searchSourceBuilder.query(queryBuilder);
        searchSourceBuilder.from((pageNum-1)*pageSize);
        searchSourceBuilder.size(pageSize);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = client.search(searchRequest,RequestOptions.DEFAULT);
        long total = searchResponse.getHits().getTotalHits().value;
        for (SearchHit hit : searchResponse.getHits().getHits()) 
            hitList.add(hit.getSourceAsMap());
            log.info("查询结果:", hit.getSourceAsString());
        
        pageList.setHitList(hitList);
        pageList.setTotal(total);
        return pageList;
    

    public List<Map<String,Object>> searchList(String index, List<EsQueryVo> esQueryVos) throws IOException 
        List<Map<String,Object>> hitList = new ArrayList();
        SearchRequest searchRequest = new SearchRequest(index);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
        if(esQueryVos != null && esQueryVos.size()>0)
            for (EsQueryVo esQueryVo:
                    esQueryVos) 
                clboolQueryBuilder(queryBuilder,esQueryVo);
            
        else
            queryBuilder.must(QueryBuilders.matchAllQuery());
        
        log.info("queryBuilder查询条件:", queryBuilder.toString());
        searchSourceBuilder.query(queryBuilder);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = client.search(searchRequest,RequestOptions.DEFAULT);
        long total = searchResponse.getHits().getTotalHits().value;
        for (SearchHit hit : searchResponse.getHits().getHits()) 
            hitList.add(hit.getSourceAsMap());
            log.info("查询结果:", hit.getSourceAsString());
        
        return hitList;
    



    /**
     * 处理流程
     * @param boolQueryBuilder
     * @param esQueryVo
     */
    public void clboolQueryBuilder(BoolQueryBuilder boolQueryBuilder,EsQueryVo esQueryVo)
        if(EsQueryType.term.getCode().equals(esQueryVo.getQueryType()))
            boolQueryBuilder.must(QueryBuilders.termsQuery(esQueryVo.getName(),esQueryVo.getValue()));
        else if(EsQueryType.prefix.getCode().equals(esQueryVo.getQueryType()))
            boolQueryBuilder.must(QueryBuilders.prefixQuery(esQueryVo.getName(),esQueryVo.getValue()));
        else if(EsQueryType.range.getCode().equals(esQueryVo.getQueryType()))
            if(RangeType.gt.getCode().equals(esQueryVo.getGteandlte()))
                boolQueryBuilder.must(QueryBuilders.rangeQuery(esQueryVo.getName()).gt(esQueryVo.getValue()));
            else if(RangeType.gte.getCode().equals(esQueryVo.getGteandlte()))
                boolQueryBuilder.must(QueryBuilders.rangeQuery(esQueryVo.getName()).gte(esQueryVo.getValue()));
            else if(RangeType.lt.getCode().equals(esQueryVo.getGteandlte()))
                boolQueryBuilder.must(QueryBuilders.rangeQuery(esQueryVo.getName()).lt(esQueryVo.getValue()));
            else
                boolQueryBuilder.must(QueryBuilders.rangeQuery(esQueryVo.getName()).lte(esQueryVo.getValue()));
            
        else
            log.error("不存在queryType",esQueryVo.getQueryType());
            throw new RuntimeException("不存在"+esQueryVo.getQueryType()+"queryType");
        
    

四 ：相应的运用：

import com.huanggr.common.core.constant.Constants;
import com.huanggr.common.core.domain.EsQueryVo;
import com.huanggr.common.core.domain.PageList;
import com.huanggr.common.core.enums.EsQueryType;
import com.huanggr.common.log.service.EsManageService;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;

@Component
public class ApplicationInitEsCreate implements ApplicationRunner 
    @Resource
    private EsManageService esManageService;

    @Override
    public void run(ApplicationArguments args) throws Exception 
        String JsonObject =getOperLogJson();
        esManageService.createIndex(Constants.INDEX_SYS_LOGS,JsonObject);
        List<EsQueryVo> esQueryVos = new ArrayList<>();
        EsQueryVo  esQueryVo = new EsQueryVo("operName","ad", EsQueryType.prefix.getCode(),null);
        esQueryVos.add(esQueryVo);
        PageList res = esManageService.searchListByPage(Constants.INDEX_SYS_LOGS,esQueryVos,1,10);

    



    public String getOperLogJson()
        return "\\n" +
                "  \\"properties\\":\\n" +
                "     \\"operId\\":\\n" +
                "        \\"type\\":\\"long\\"\\n" +
                "      ,\\n" +
                "     \\"businessType\\":\\n" +
                "        \\"type\\":\\"integer\\"\\n" +
                "      ,\\n" +
                "     \\"businessTypes\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"method\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"requestMethod\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"operatorType\\":\\n" +
                "        \\"type\\":\\"integer\\"\\n" +
                "      ,\\n" +
                "     \\"operName\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"deptName\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"operUrl\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"operIp\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"operParam\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"jsonResult\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"status\\":\\n" +
                "        \\"type\\":\\"integer\\"\\n" +
                "      ,\\n" +
                "     \\"errorMsg\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "      ,\\n" +
                "     \\"startDate\\": \\n" +
                "     \\"format\\": \\"yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis\\",\\n" +
                "     \\"type\\": \\"date\\"\\n" +
                "      ,\\n" +
                "     \\"costTime\\":\\n" +
                "        \\"type\\":\\"long\\"\\n" +
                "      ,\\n" +
                "     \\"title\\":\\n" +
                "        \\"type\\":\\"keyword\\"\\n" +
                "     \\n" +
                "  \\n" +
                "";
    

import com.alibaba.fastjson2.JSON;
import com.huanggr.common.core.constant.Constants;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import com.huanggr.common.core.constant.SecurityConstants;
import com.huanggr.system.api.RemoteLogService;
import com.huanggr.system.api.domain.SysOperLog;

import javax.annotation.Resource;
import java.util.Date;
import java.util.HashMap;
import java.util.UUID;

/**
 * 异步调用日志服务
 *
 * @author huanggr
 */
@Service
public class AsyncLogService

    @Autowired
    private RemoteLogService remoteLogService;
    @Resource
    private EsManageService esManageService;


    /**
     * 保存系统日志记录
     */
    @Async
    public void saveSysLog(SysOperLog sysOperLog)
    
        remoteLogService.saveLog(sysOperLog, SecurityConstants.INNER);
    


    @Async
    public  void saveSysLogEs(SysOperLog sysOperLog) throws Exception 
        sysOperLog.setOperTime(new Date());
        HashMap<String,Object> map = JSON.parseObject(JSON.toJSONString(sysOperLog), HashMap.class);
        esManageService.createDocumentByHashMap(map, Constants.INDEX_SYS_LOGS, UUID.randomUUID().toString());