单词 统计续（补）

Posted 2022-01-14 goubb

短语统计

我们在处理文本的时候只需将提取出来的文本数据进行特殊的分割处理，比如只需将英文的“,”，“.”，“？”，“!"以及回车符设为分隔符。并将一些无用单词作为间断比如

"a",  "it", "the", "and", "this"等。

package analyse_word;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Scanner;
import java.util.Set;

public class recognize_sentence
 
 public static boolean useless(String str) throws FileNotFoundException
  File file = new File("D:\\useless.txt");// 读取文件
  String words[] = new String [100000];
  int out_words[] = new int [100000];
  if (!file.exists()) // 如果文件打不开或不存在则提示错误
   System.out.println("文件不存在");
   return false;
  
  Scanner x = new Scanner(file);
  HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
  while (x.hasNextLine())
   String line = x.nextLine();
   String[] lineWords = line.split("[\n]");
   Set<String> wordSet = hashMap.keySet();
   for (int i = 0; i < lineWords.length; i++)
    if (wordSet.contains(lineWords[i]))
     Integer number = hashMap.get(lineWords[i]);
     number++;
     hashMap.put(lineWords[i], number);
     else
     hashMap.put(lineWords[i], 1);
    
   
  
  Iterator<String> iterator = hashMap.keySet().iterator();
  int max = 0,i=0;
  while (iterator.hasNext())
   String word = iterator.next();
   if(!"".equals(word)&&word!=null&&!"a".equals(word)&&!"the".equals(word)&&!"　　".equals(word))
    if(str.indexOf(" "+word+" ")==0)
     return true;
    
    words[i]=word;
    out_words[i]=hashMap.get(word);
    i++;
   
  
  return true;
 

 public static void recognize() throws FileNotFoundException
  File file = new File("D:\\Englis_letters.txt");// 读取文件
  if (!file.exists()) // 如果文件打不开或不存在则提示错误
   System.out.println("文件不存在");
   return;
  
  Scanner x = new Scanner(file);
  HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
  while (x.hasNextLine())
   String line = x.nextLine();
   String[] lineWords = line.split("[\\t+;.,“”‘’?!\n+]");
   Set<String> wordSet = hashMap.keySet();
   for (int i = 0; i < lineWords.length; i++)
    if (wordSet.contains(lineWords[i]))
     Integer number = hashMap.get(lineWords[i]);
     number++;
     hashMap.put(lineWords[i], number);
     else
     hashMap.put(lineWords[i], 1);
    
   
  
  Iterator<String> iterator = hashMap.keySet().iterator();
  while (iterator.hasNext())
   String word = iterator.next();
   if(useless(word))
    System.out.println(word);
   
  
 

 public static void main(String[] args) throws FileNotFoundException
  recognize();