java Solr查询和Java

Posted 2021-05-08

package au.edu.parentPortal.controllers;

import au.edu.parentPortal.dao.UrlBasedContentRepository;
import au.edu.parentPortal.exceptions.DaoException;
import au.edu.parentPortal.exceptions.ServiceNotAvailableException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by hveluchamy on 22/10/2015.
 */
@Controller
public class SearchController extends BaseController{
    private final Logger LOG = LogManager.getLogger(SearchController.class);

    @Autowired
    private UrlBasedContentRepository customSolrContentRepository;
    
    public SearchController (){}
    public SearchController (UrlBasedContentRepository customSolrContentRepository){
    	
    }

    @RequestMapping(value="/searchContent", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
    @ResponseBody
    public String getSearchResult(@RequestParam(value = "searchInput", required = false) String searchInput, @RequestParam("yearLevel") String yearLevel, @RequestParam String sortBy, @RequestParam boolean asc, @RequestParam int startIndex, @RequestParam String filterQuery, @RequestParam int numPerPage) throws Exception {
        List<String> stringList = new ArrayList<>();
        try {
            return customSolrContentRepository.search(searchInput, yearLevel,sortBy, filterQuery,asc, startIndex, numPerPage).toString();
        } catch (DaoException e) {
            throw new ServiceNotAvailableException("searchFilter", "SOLR Issue: " + e.toString());
        }
    }



}

# Working mode
debug.mode=true

hibernate.showSql=false

#csv-file location
parent.csv.filePath=c:/csv-files
#solr popularity file locaion
solr.popularity.file.path=C:/solr-5.3.1/server/solr/parentswebsite/data/external_popularity_views.txt
solr.popularity.job.cron=0 0/20 * 1/1 * ? *

SOLR=http://localhost:8983/solr/parentswebsite
Solr.host.name=localhost:8983

#Solr-settings and queries
Solr.core.with.query.type=/solr/parentswebsite/select
search.with.filter.term=q=%1$s&wt=json&indent=true&bq=content_type:ACTIVITY^5&qf=useful_terms^50+brief_description^15+title^10+_text_&facet=true&facet.field=capability&facet.field=facet_year_level&facet.field=content_type&facet.sort=index&facet.mincount=1&fl=title,display_year_level,capability,brief_description,useful_terms,content_type,internal_id,ac_code,topic,img_filename,resource_id,score&defType=dismax&%2$s&facet.sort=index&json.nl=map&%3$s&%4$s&%5$s&tie=0.9

#solr-glossary-core-settings and queries
solr.glossary.core.with.query.type=/solr/pwsiteglossarylinks/select
get.glossary.solr.content=q=*:*&fq=content_type:%1$s&rows=250&wt=json&indent=true
#fq=content_type%3AGlossary &fq=content_type:ACTIVITY&

search.more.like.this=q=internal_id:%1$s&wt=json&indent=true&fl=title,display_year_level,topic,useful_terms,content_type,internal_id,img_filename,resource_id,score&json.nl=map&mlt=true&mlt.fl=%2$s&mlt.mindf=1&mlt.mintf=1&mlt.count=10&tie=0.9
search.popular.game=q=%1$s AND NOT internal_id:%3$s&fq=capability:%2$s&fq=content_type:GAME&sort=popularity_views+desc&wt=json&indent=true&tie=0.9
search.popular.activity=q=%1$s&fq=capability:%2$s&fq=content_type:ACTIVITY&fq=facet_year_level:%3$s&sort=popularity_views+desc&wt=json&indent=true&tie=0.9

search.featured.list=q=*:*&fq=featured:true&rows=100&sort=internal_id asc&wt=json&indent=true&tie=0.9

list.solr.content=q=*:*&fq=content_type:%1$s&wt=json&indent=true&sort=%2$s&start=%3$s&rows=%4$s&facet.field=topic&facet.sort=index&json.nl=map&facet=true&tie=0.9
get.solr.content.by.id=q=%1$s&fq=content_type:%2$s&wt=json&indent=true&json.nl=map&tie=0.9

#Solr-indexing-setting and url
Solr.core.with.query.import=/solr/parentswebsite/dataimport
solr.full.import=command=full-import&clean=true&optimize=true&entity=pwdata&commit=true&
solr.delta.import=command=delta-import&optimize=true&entity=pwdata&commit=true

scootle.data.import.url=http://localhost:8085/service/availableobjects/getlist
#scootle.data.import.url=http://192.168.75.18:8080/scootle-api/service/availableobjects/getlist


#cron expression enabled = true to run job
scootle.job.enabled=true
#cron expression for scootle nightly job
scootle.lookup.job.cron=0 0/15 * 1/1 * ? *
#every min
#0 0/1 * 1/1 * ? *
#every 12 hrs
#0 0 12 1/1 * ? *h

package au.edu.parentPortal.controllers;

import au.edu.parentPortal.beans.EverydayIdea;
import au.edu.parentPortal.dao.SolrContentRepository;
import au.edu.parentPortal.dao.UrlBasedContentRepository;
import au.edu.parentPortal.enums.EntityType;
import au.edu.parentPortal.exceptions.DaoException;
import au.edu.parentPortal.exceptions.ManagerException;
import au.edu.parentPortal.exceptions.NotFoundException;
import au.edu.parentPortal.exceptions.ServiceNotAvailableException;
import au.edu.parentPortal.manager.EverydayIdeaManager;
import au.edu.parentPortal.util.ParentPortalProperties;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.util.Assert;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;

import javax.servlet.http.HttpServletRequest;
import java.util.List;

/**
 * Created by hveluchamy on 5/10/2015.
 */
@Controller
public class EverydayIdeaController extends BaseController {
    private final Logger LOG = LogManager.getLogger(EverydayIdeaController.class);

    @Autowired
    private EverydayIdeaManager everydayIdeaManager;

    @Autowired
    private SolrContentRepository solrContentRepository;

    @Autowired
    private ParentPortalProperties parentPortalProperties;

    @Autowired
    private UrlBasedContentRepository customSolrContentRepository;

    @Autowired
    private HttpServletRequest request;
    
    
    public EverydayIdeaController (){}

    public EverydayIdeaController (EverydayIdeaManager everydayIdeaManager , SolrContentRepository solrContentRepository, UrlBasedContentRepository customSolrContentRepository, HttpServletRequest request){
    	this.solrContentRepository= solrContentRepository;
    	this.everydayIdeaManager = everydayIdeaManager;
        this.customSolrContentRepository = customSolrContentRepository;
        this.request = request;

    }
    
    
    @RequestMapping(value="/everydayIdea", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
     @ResponseBody
     public String getEverydayIdeaById(@RequestParam("id") String id) throws Exception {
        Assert.notNull(id);
        EverydayIdea everydayIdea;
        try{
            if (!(request.getHeader("user-agent").indexOf("Googlebot") >= 0)){
                everydayIdeaManager.getEverydayIdeaAndStoreStat(id);
            }
            return customSolrContentRepository.getSolrContentById( id,"IDEA").toString();
        }catch (DaoException e) {
            throw new ServiceNotAvailableException("everydayIdea.id", "Database Issue: " + e.toString());
        }
    }


    @RequestMapping(value="/list/everydayIdea", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
    @ResponseBody
    public String getSolrEverydayIdeas(
    		@RequestParam String sortBy,
    		@RequestParam boolean asc,  
    		@RequestParam int startIndex, 
    		@RequestParam int numPerPage,
            @RequestParam String filterQuery) throws Exception {
        try{
            if(StringUtils.isEmpty(sortBy)) {
                sortBy="title_str";
            }
            return customSolrContentRepository.listSolrContent("IDEA",sortBy, asc,startIndex ,numPerPage, filterQuery).toString();
        }catch (DaoException e) {
            throw new ServiceNotAvailableException("everydayIdeaList", "Database Issue: " + e.toString());
        }
    }


    @RequestMapping(value="/everydayIdeas", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
    @ResponseBody
    public List<EverydayIdea> getEverydayIdeas() throws Exception {
        List<EverydayIdea> everydayIdeas;
        try{
            everydayIdeas = everydayIdeaManager.getAllIdeas();
            if(everydayIdeas == null) {
                throw new NotFoundException("everydayIdeas.id", "Everydayideas list is empty");
            }
        }catch (ManagerException e) {
            throw new ServiceNotAvailableException("everydayIdeaList", "Database Issue: " + e.toString());
        }

        return everydayIdeas;
    }

    @RequestMapping(value="/otherIdeas", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
    @ResponseBody
    public String getOtherIdeas(@RequestParam("id") String id) throws Exception {
        Assert.notNull(id);
        try {
            return customSolrContentRepository.searchOtherContents(id, EntityType.EVERYDAY_IDEA, "topic").toString();

        } catch (DaoException e) {
            throw new ServiceNotAvailableException("otherIdeas", "SOLR Issue: " + e.toString());
        }
    }


    @RequestMapping(value="/list/featureList", method = RequestMethod.GET, produces = "application/json;charset=UTF-8")
    @ResponseBody
    public String getSolrFeaturedList() throws Exception {
        try{
            return customSolrContentRepository.listFeaturedContent().toString();
        }catch (DaoException e) {
            throw new ServiceNotAvailableException("featureList", "Database Issue: " + e.toString());
        }
    }



}

* The dis max has been used to give weightage to field
* more like this is a build in property of solr

package au.edu.parentPortal.dao.hibernate;

import au.edu.parentPortal.dao.UrlBasedContentRepository;
import au.edu.parentPortal.enums.EntityType;
import au.edu.parentPortal.exceptions.DaoException;
import au.edu.parentPortal.util.ParentPortalProperties;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Set;
import java.util.StringTokenizer;

/**
 * Created by hveluchamy on 23/10/2015.
 */
public class UrlBasedContentRepositoryImpl implements UrlBasedContentRepository{
    private final Logger LOG = LogManager.getLogger(UrlBasedContentRepositoryImpl.class);

    @Autowired
    ParentPortalProperties parentPortalProperties;

    private static final String ACTIVITY_ID_SOLR_PREFIX = "A";
    private static final String IDEA_ID_SOLR_PREFIX = "I";
    public JsonElement  search(String searchTerm, String yearLevel, String sortBy, String filterQuery, boolean asc, int startIndex, int numPerPage) throws IOException, DaoException {
    	
        URI uri = null;
        String filterTerm = null;

        if(yearLevel.toLowerCase().trim().equals("all-years")) {
            filterTerm=null;
        } else {
            filterTerm = "fq=facet_year_level:" + "\"" + yearLevel + "\"";
        }

        filterTerm = filterTerm + "&" + filterQuery;

        if (!StringUtils.isEmpty(sortBy)){
        	sortBy = "sort="+sortBy+"+";
        	if (asc) {
    			sortBy += "asc";
    		} else if (!asc) {
    			sortBy += "desc";
    		}
        }
        
        //set start and end index
        String startRange = "start="+startIndex;
        String endRange="rows="+numPerPage;

        //get all if search term is empty
        if(StringUtils.isEmpty(searchTerm)) {
            searchTerm = "ACTIVITY OR GAME OR IDEA";
        }

        String parametrizedQuery= String.format(parentPortalProperties.getSEARCHWITHFILTERTERM(), searchTerm, sortBy,filterTerm,startRange,endRange );

        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQuery,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            //JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent()));
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent(), StandardCharsets.UTF_8));//Convert the input stream to a json element
            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }


    //mlt - more like this
    //For activity mltField is usefultTerms. For Ideas its topic
    public JsonElement searchOtherContents(String contentId, EntityType contentType, String mltField) throws IOException, DaoException {
        URI uri = null;

        /*if(contentType == EntityType.ACTIVITY) {
            contentId = ACTIVITY_ID_SOLR_PREFIX + contentId;
        } else if (contentType == EntityType.EVERYDAY_IDEA) {
            contentId = IDEA_ID_SOLR_PREFIX + contentId;
        }*/

        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getSEARCHMORELIKETHIS(), contentId, mltField);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent())); //Convert the input stream to a json element
            JsonObject rootObject = root.getAsJsonObject();

            LOG.info(rootObject.getAsJsonObject("moreLikeThis"));
            LOG.info(rootObject.getAsJsonObject("moreLikeThis").getAsJsonObject(contentId));
            JsonElement root1 = rootObject.getAsJsonObject("moreLikeThis").getAsJsonObject(contentId);
            return root1;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

    @Override
    public JsonElement listSolrContent(String contentType,String sortBy, boolean asc, int startIndex, int numPerPage, String filterQuery) throws IOException, DaoException {    	
    	
        URI uri = null;      
        if (!StringUtils.isEmpty(sortBy)){
        	sortBy = sortBy+"+";
        	if (asc) {
    			sortBy += "asc";
    		} else if (!asc) {
    			sortBy += "desc";
    		}
        }
        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getLISTSOLRCONTENT(),contentType, sortBy, startIndex, numPerPage);
        parametrizedQueryWithFilter=parametrizedQueryWithFilter + "&" + filterQuery;
        LOG.info(parametrizedQueryWithFilter);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent())); //Convert the input stream to a json element
            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

    @Override
    public JsonElement getSolrContentById(String internalId, String contentType) throws IOException, DaoException {
        internalId = "internal_id:" + internalId;
        URI uri = null;

        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getGETSOLRCONTENTBYID(),internalId, contentType);
        LOG.info(parametrizedQueryWithFilter);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent(), StandardCharsets.UTF_8));
            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

    @Override
    public JsonElement listFeaturedContent() throws IOException, DaoException {

        URI uri = null;

        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getLISTFEATUREDCONTENT());
        LOG.info(parametrizedQueryWithFilter);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent())); //Convert the input stream to a json element
            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

    @Override
    public JsonElement listGlossaryContent(String contentType) throws IOException, DaoException {
        URI uri = null;
        String parametrizedQuery = String.format(parentPortalProperties.getGETGLOSSARYSOLRCONTENT(), contentType);
        try {
            uri = new URI("http",
                       parentPortalProperties.getSOLRHOSTNAME(),
                       parentPortalProperties.getSOLRGLOSSARYCOREANDQUERYTYPE(),
                       parametrizedQuery,
                       null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent())); //Convert the input stream to a json element
            return root;

        } catch (Exception e) {
            throw new DaoException("Error getting glossary solr results: " + e);
        }
    }

    public JsonElement searchPopularGamesByCapability(String capability, String yearLevel, String currentId) throws IOException, DaoException {
        URI uri = null;
        capability = capability.replaceAll(",", " ");

        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getSEAERCHPOPULARGAME(), processYearLevel(yearLevel), capability, currentId);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent(), StandardCharsets.UTF_8));

            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

    private String processYearLevel(String yearLevel) {
        //facet_year_level:%3$s
        try {
            yearLevel = yearLevel.replace("\"", "");
            HashMap<String, Integer> yearGroups = new HashMap<String, Integer>();
            StringTokenizer st = new StringTokenizer(yearLevel, ",");
            String returnYearString="";
            while (st.hasMoreTokens()){
                String yearGroupStr = st.nextToken();
                yearGroupStr = "year:" + "\"" + yearGroupStr + "\"";
                yearGroups.put(yearGroupStr, 1);
            }
            Set<String> years = yearGroups.keySet();
            for (String year: years){
                year = year + " OR ";
                returnYearString = returnYearString + year;
            }
            return (returnYearString == null || "".equals(returnYearString) ) ? null : returnYearString.substring(0, returnYearString.length()-4) ;
        } catch (Exception e) {
            LOG.error(e);
        }
        return null;
    }

    public JsonElement searchPopularActivityByCapability(String capability, String yearLevel) throws IOException, DaoException {
        URI uri = null;
        capability = capability.replaceAll(",", " ");
        //Needed to do this for SOLR activity internalId
        LOG.info(parentPortalProperties.getSEAERCHPOPULARGAME());        
        String parametrizedQueryWithFilter= String.format(parentPortalProperties.getSEARCHPOPULARACTIVITY(), "*:*", capability, yearLevel);
        try {
            uri = new URI("http",
                    parentPortalProperties.getSOLRHOSTNAME(),
                    parentPortalProperties.getSOLRCOREANDQUERYTYPE(),
                    parametrizedQueryWithFilter,
                    null);
            URL url = uri.toURL();
            LOG.info(url);
            HttpURLConnection request = null;
            request = (HttpURLConnection) url.openConnection();
            request.setRequestMethod("GET");
            request.setRequestProperty("Accept-Charset", "UTF-8");
            request.connect();
            JsonParser jp = new JsonParser(); //from gson
            JsonElement root = jp.parse(new InputStreamReader((InputStream) request.getContent(), StandardCharsets.UTF_8)); //Convert the input stream to a json element

            return root;
        } catch (Exception e) {
            throw new DaoException("Error getting solr results: " + e);
        }

    }

}