42lucene和机器学习进行全文搜索，并排序

Posted 2020-08-22 香港胖仔

package com.lucene.test;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.joone.engine.FullSynapse;
import org.joone.engine.LinearLayer;
import org.joone.engine.Monitor;
import org.joone.engine.NeuralNetEvent;
import org.joone.engine.NeuralNetListener;
import org.joone.engine.SigmoidLayer;
import org.joone.engine.learning.TeachingSynapse;
import org.joone.io.MemoryInputSynapse;
import org.joone.io.MemoryOutputSynapse;
import org.joone.net.NeuralNet;
import org.junit.Test;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.lucene.domain.Article;

public class TestLucene implements NeuralNetListener{
    private NeuralNet nnet = null;
    private MemoryInputSynapse inputSynapse,desireOutputSynapse;
    LinearLayer input;
    SigmoidLayer hidden,output;
    boolean singleThreadMode = true;
    
    //XOR input
    private double[][] inputArray = new double[][]{
            {0.0,0.0},
            {0.0,1.0},
            {1.0,0.0},
            {1.0,1.0}
    };
    
    //XOR desired output
    private double[][] desiredOutputArray = new double[][]{
            {0.0},
            {1.0},
            {1.0},
            {1.0}
    };
    
    /**
     * 创建索引
     * @throws Exception
     */
    @Test
    public void testCreateIndex() throws Exception{
        int fileNum = 1;
        List<String> contents = new ArrayList<String>();
        InputStream inputStream = null;
        String value = null;
        File directory = new File("./20_newsgroups");
        if(directory.isDirectory()){
            File[] files = directory.listFiles();
            for (int i = 0; i < 1; i++) {
                if(files[i].isDirectory()){
                    File[] subFiles = files[i].listFiles();
                    for (int j = 0; j < 10; j++) {
                        inputStream = new BufferedInputStream(new FileInputStream(subFiles[j]));
                        StringBuffer tempContent = new StringBuffer();
                        byte[] bytes = new byte[1024*10];
                        int len = 0;
                        while((len = inputStream.read(bytes))!=-1){
                            tempContent = tempContent.append(new String(bytes));
                        }
                        value = tempContent.toString();
                        System.out.println(value);
                        inputStream.close();
                        Article article = new Article(fileNum,subFiles[j].getName(),tempContent.toString());
                        Directory saveDirectory = FSDirectory.open(Paths.get("./indexDir/"));
                        //分词器
                        Analyzer analyzer = new WhitespaceAnalyzer();
                        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
                        Document doc = new Document();
                        doc.add(new TextField("id", article.getId().toString(), Store.YES));
                        doc.add(new TextField("title", article.getTitle(), Store.YES));
                        doc.add(new TextField("content", article.getContent(), Store.YES));
                        IndexWriter indexWriter = new IndexWriter(saveDirectory,iwc);
                        System.out.println("have already add file to fileDocment system"+fileNum);
                        indexWriter.addDocument(doc);
                        indexWriter.close();//释放资源
                        fileNum = fileNum+1;
                    }
                }
            }
        }
        
        //1.将需要添加的实体构造成实体对象
        Article article = new Article(1,"Lucene是全文检索框架",
                "全文检索(Full-Test Retrieval)是以文本作为检索对象，找出含有指定词汇的文本。"+
                "全面，准确和快速是衡量全文检索系统的关键指标。");
        
        //2，保存到数据库（此步骤暂时省略）
        
        //3、建立索引(lucene)
        //索引库目录
        
        
        
        //将 Article 转换为Document
        
        
        
        
        //保存到索引库中
        
    }
    
    /**
     * 测试搜索
     * @throws IOException 
     * @throws ParseException 
     */
    @Test
    public void testSearch() throws IOException, ParseException{
        //1、搜索条件
        String queryCondition = "philosophical";
        
        //2、执行搜索(lucene)
        List<Article> articles = new ArrayList<Article>();
        
        //----------搜索代码------------------------
        Directory directory = FSDirectory.open(Paths.get("./indexDir/"));
        Analyzer analyzer = new WhitespaceAnalyzer();//创建分词器
        
        //把查询字符串转换为Query对象(只在title中查询)
        QueryParser queryParser = new QueryParser("content",analyzer);
        Query query = queryParser.parse(queryCondition);
        
        //2执行搜索得到搜索结果
        IndexReader indexReader = DirectoryReader.open(directory);
        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
        TopDocs topDocs = indexSearcher.search(query, 100);
        
        Integer count = topDocs.totalHits;//总结果数量
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;//返回前N条结果
        
        //2.3处理结果
        for (int i = 0; i < scoreDocs.length; i++) {
            ScoreDoc scoreDoc= scoreDocs[i];
            int docId = scoreDoc.doc;
            System.out.println("得分是："+scoreDoc.score+"内部编号是:"+docId);
            
            //根据内部编号取出真正的Document数据
            Document doc = indexSearcher.doc(docId);
            
            //将document转化为Article
            Article article = new Article(Integer.parseInt(doc.get("id")),doc.get("title"),doc.get("content"));
            articles.add(article);
        }
        
        //------------------------------------------
        //3、控制台显示结果
        System.err.print("总结果数:"+count);
        for (Article article : articles) {
            System.out.println("查询结果：ID为："+article.getId()+"，title为："+article.getTitle());
        }
        indexSearcher.getIndexReader().close();
    }
    
    
    @Test
    public void testNeuralNet(){
        TestLucene testLucene = new TestLucene();
        testLucene.initNeuralNet();
        testLucene.train();
        testLucene.interrogate();
    }
    
    public void initNeuralNet(){
        //First create the three layers
        input = new LinearLayer();
        hidden = new SigmoidLayer();
        output = new SigmoidLayer();
        
        //set the dimensions of the layers
        input.setRows(2);
        hidden.setRows(3);
        output.setRows(1);
        
        input.setLayerName("L.input");
        hidden.setLayerName("L.hidden");
        output.setLayerName("L.output");
        
        //Now create the two Synapses
        FullSynapse synapse_IH = new FullSynapse();//input -->hidden conn
        FullSynapse synapse_HO = new FullSynapse();//hidden -->output conn
        
        //Connect the input layer whit the hidden layer
        input.addOutputSynapse(synapse_IH);
        hidden.addInputSynapse(synapse_IH);
        
        //Connect the hidden layer whit the output layer
        hidden.addOutputSynapse(synapse_HO);
        output.addInputSynapse(synapse_HO);
        
        //the input to the neural net
        inputSynapse = new MemoryInputSynapse();
        input.addInputSynapse(inputSynapse);
        
        //The Trainer and its desired output
        desireOutputSynapse = new MemoryInputSynapse();
        TeachingSynapse trainer = new TeachingSynapse();
        
        trainer.setDesired(desireOutputSynapse);
        
        //Now we add this structure to a NeuralNet object
        nnet = new NeuralNet();
        
        nnet.addLayer(input,NeuralNet.INPUT_LAYER);
        nnet.addLayer(hidden,NeuralNet.HIDDEN_LAYER);
        nnet.addLayer(output, NeuralNet.OUTPUT_LAYER);
        nnet.setTeacher(trainer);
        output.addOutputSynapse(trainer);
        nnet.addNeuralNetListener(this);
    }
    
    public void train(){
        //set the inputs
        inputSynapse.setInputArray(inputArray);
        inputSynapse.setAdvancedColumnSelector("1,2");
        //set the desired outputs
        desireOutputSynapse.setInputArray(desiredOutputArray);
        desireOutputSynapse.setAdvancedColumnSelector("1");
        //get the monitor object to train or feed forward
        Monitor monitor = nnet.getMonitor();
        
        //set the monitor parameters
        monitor.setLearningRate(0.8);
        monitor.setMomentum(0.3);
        monitor.setTrainingPatterns(inputArray.length);
        monitor.setTotCicles(5000);
        monitor.setLearning(true);
        
        long initms = System.currentTimeMillis();
        //Run the network in single-thread,synchronized mode
        nnet.getMonitor().setSingleThreadMode(singleThreadMode);
        nnet.go(true);
        System.out.println("Total time="+(System.currentTimeMillis()-initms)+"ms");
    }

    
    public void interrogate(){
        double[][] inputArray = new double[][]{
                {0.0,1.0},
                {1.0,0.0},
                {1.0,1.0},
                {0.0,0.0}
        };
        //set the inputs
        inputSynapse.setInputArray(inputArray);
        inputSynapse.setAdvancedColumnSelector("1,2");
        Monitor monitor = nnet.getMonitor();
        monitor.setTrainingPatterns(4);
        monitor.setTotCicles(1);
        monitor.setLearning(false);
        MemoryOutputSynapse memOut = new MemoryOutputSynapse();
        //set the output synapse to write the output of the net
        
        if(nnet != null){
            nnet.addOutputSynapse(memOut);  
            System.out.println(nnet.check());
            nnet.getMonitor().setSingleThreadMode(singleThreadMode);
            nnet.go();
            for (int i = 0; i < 4; i++) {
                double[] pattern = memOut.getNextPattern();
                System.out.println("Output pattern #"+(i+1)+"="+pattern[0]);
            }
            System.out.println("Interrogating Finished");
        }
    }
    
    
    public void cicleTerminated(NeuralNetEvent arg0) {
        
    }

    public void errorChanged(NeuralNetEvent e) {
        Monitor mon=(Monitor) e.getSource();
        if(mon.getCurrentCicle()%100==0){
            System.out.println("Epoch:"+(mon.getTotCicles()-mon.getCurrentCicle())+"RMSE:"
                    +mon.getGlobalError());
        }
    }

    public void netStarted(NeuralNetEvent e) {
        Monitor mon = (Monitor) e.getSource();
        System.out.println("Network started for ");
        if(mon.isLearning()){
            System.out.println("training");
        }else{
            System.out.println("interrogation.");
        }
    }

    public void netStopped(NeuralNetEvent e) {
        Monitor mon = (Monitor) e.getSource();
        System.out.println("Network stopped . Last RMSE="
                +mon.getGlobalError());
    }

    public void netStoppedError(NeuralNetEvent e, String error) {
        System.out.println("Network stopped due the following error:"
                +error);
    }
}

结果

得分是：0.25462872内部编号是:7840
得分是：0.24006625内部编号是:7841
查询结果：ID为：2，title为：51060总结果数:2
查询结果：ID为：1，title为：49960