ElasticsearchRestTemplate实现scroll分页

Posted 2022-01-19 编程老高

UserInfo类：

package com.jd.vo;

import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;

/**
 * @Document 用来声明ES索引信息
 * indexName 索引名称
 * shards 主分区数量，默认5
 * replicas 副本分区数量，默认1
 * createIndex true：SpringBoot启动后自动创建索引，此时映射Mapping为空，默认为true；false：SpringBoot启动后不会自动创建索引，需要手动创建索引。
 */
@Document(indexName = "userinfo_index", shards = 5, replicas = 1, createIndex = false)
public class UserInfo 

    @Id
    private Integer id;
    @Field(type = FieldType.Keyword)
    private String name;
    //type : 字段数据类型
    //analyzer : 分词器类型
    //index : 是否索引，默认为true
    @Field(type = FieldType.Text, analyzer = "ik_max_word", index = true)
    private String address;

    public UserInfo(Integer id, String name, String address) 
        this.id = id;
        this.name = name;
        this.address = address;
    

    public Integer getId() 
        return id;
    
    public void setId(Integer id) 
        this.id = id;
    
    public String getName() 
        return name;
    
    public void setName(String name) 
        this.name = name;
    
    public String getAddress() 
        return address;
    
    public void setAddress(String address) 
        this.address = address;
    

PagingTests类：

package com.jd;

import com.jd.vo.UserInfo;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchScrollHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

@SpringBootTest
public class PagingTests 

    @Autowired
    private ElasticsearchRestTemplate template;

    //批量操作，bulk [bʌlk] n. 巨大的体重(或重量、形状、身体等);
    @Test
    public void bulk() throws IOException 
        List<UserInfo> list = new ArrayList<>();
        list.add(new UserInfo(1, "Jim", "河南郑州市高新技术开发区朗月公园茂"));
        list.add(new UserInfo(2, "Lucy", "内蒙古自治区呼和浩特市回民区万嘉康城"));
        list.add(new UserInfo(3, "Kate", "江西省南昌市红谷滩区天赐良园"));
        list.add(new UserInfo(4, "Jhon", "福建省厦门市湖里区碧湖嘉园"));
        list.add(new UserInfo(5, "Tom", "吉林省长春市宽城区中正御华庭"));
        list.add(new UserInfo(6, "Jimmy", "广东省东莞市东城街道天宝花园"));
        list.add(new UserInfo(7, "Robert", "新疆自治区乌鲁木齐市头屯河区乌西五街小区"));
        Iterable iterable = template.save(list);
        System.out.println(iterable);
    
    
    @Test
    public void scroll1() throws IOException 
        // 设置每页数据量
        int pageSize = 2;
        MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();//条件
        FieldSortBuilder sortBuilder = new FieldSortBuilder("id").order(SortOrder.ASC);//排序
        NativeSearchQuery nativeSearchQuery = new NativeSearchQueryBuilder().withQuery(queryBuilder).withSort(sortBuilder).build();
        nativeSearchQuery.setMaxResults(pageSize);// 设置每页数据量

        long scrollTimeInMillis = 60 * 1000;//设置缓存内数据的保留时间，不要把缓存时时间设置太长，否则占用内存。
        // 1、缓存第一页符合搜索条件的数据
        SearchScrollHits<UserInfo> searchScrollHits = template.searchScrollStart(scrollTimeInMillis, nativeSearchQuery, UserInfo.class, IndexCoordinates.of("userinfo_index"));
        String scrollId = searchScrollHits.getScrollId();

        int scrollTime=1;
        while (searchScrollHits.hasSearchHits()) //2、判断searchScrollHits中是否有命中数据，如果为空，则表示已将符合查询条件的数据全部遍历完毕
            System.out.println("第"+scrollTime+"页数据，数据总数：" + searchScrollHits.getSearchHits().size());

            for (SearchHit<UserInfo> searchHit : searchScrollHits.getSearchHits()) //3、 从缓存中读取数据
                UserInfo book = searchHit.getContent();
                System.out.println(book.getId()+" "+book.getName()+" "+book.getAddress());
            
            //4、根据上次搜索结果scroll_id进入下一页数据搜索
            searchScrollHits = template.searchScrollContinue(scrollId, scrollTimeInMillis, UserInfo.class, IndexCoordinates.of("userinfo_index"));//该方法执行后将重新刷新快照保留时间
            scrollId = searchScrollHits.getScrollId();
            scrollTime=scrollTime+1;
        

        List<String> scrollIds = new ArrayList<>();
        scrollIds.add(scrollId);
        template.searchScrollClear(scrollIds);// 清除 scroll
    

    @Test
    public void scroll2() throws IOException 
        int pageNo=4;//设置第几页
        int pageSize = 2;// 设置每页数据量

        MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();//条件
        FieldSortBuilder sortBuilder = new FieldSortBuilder("id").order(SortOrder.ASC);//排序
        NativeSearchQuery nativeSearchQuery = new NativeSearchQueryBuilder().withQuery(queryBuilder).withSort(sortBuilder).build();
        nativeSearchQuery.setMaxResults(pageSize);// 设置每页数据量

        long scrollTimeInMillis = 60 * 1000;//设置缓存内数据的保留时间
        //1、缓存第一页符合搜索条件的数据
        SearchScrollHits<UserInfo> searchScrollHits = template.searchScrollStart(scrollTimeInMillis, nativeSearchQuery, UserInfo.class, IndexCoordinates.of("userinfo_index"));
        String scrollId = searchScrollHits.getScrollId();

        int scrollTime=1;
        while (searchScrollHits.hasSearchHits() && scrollTime<pageNo) //2、判断searchScrollHits中是否有命中数据，如果为空，则表示已将符合查询条件的数据全部遍历完毕
            //3、根据上次搜索结果scroll_id进入下一页数据搜索
            searchScrollHits = template.searchScrollContinue(scrollId, scrollTimeInMillis, UserInfo.class, IndexCoordinates.of("userinfo_index"));//该方法执行后会重新刷新快照保留时间
            scrollId = searchScrollHits.getScrollId();
            scrollTime=scrollTime+1;
        

        List<String> scrollIds = new ArrayList<>();
        scrollIds.add(scrollId);
        template.searchScrollClear(scrollIds);// 清除 scroll

        for (SearchHit<UserInfo> searchHit : searchScrollHits.getSearchHits()) //4、 从缓存中读取数据
            UserInfo userInfo = searchHit.getContent();
            System.out.println(userInfo.getId()+" "+userInfo.getName()+" "+userInfo.getAddress());