贝叶斯实现拼写检查器

Posted 2020-10-15

import re,collections

# 把语料中的单词全部抽取出来，转成小写，并且取出单词中间的特殊符号
def words(text):
  return re.findall(‘[a-z]+‘,text.lower())

def train(features):
  model = collections.defaultdict(lambda:1) # 词频的默认出现数为1
  for f in features:
    model[f] += 1
  return model

NWORDS = train(words(open(big.txt‘).read()))

apphabet = ‘abcdefghijklmnopqrstuvwxyz‘

# 返回所有与单词w编辑距离为1的集合
def editsl(word):
  n = len(word)
  return set([word[0:i]+word[i+1:] for i in range(n)] +                          # deletion
             [word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)] +      # transposition
             [word[0:i]+c+word[i+1:] for i in range(n) for c in apphabet] +      # alteration
             [word[0:i]+c+word[i:] for i in range(n+1) for c in apphabet])       # insertion

# 返回所有与单词w编辑距离为2的集合
# 在这些编辑距离小于2的词中间，只把那些正确的词作为候选词
def edits2(word):
  return set(e2 for e1 in editsl(word) for e2 in edits1(el) if e2 in NWORDS)

def known_edits2(words):return set(w for w in words if w in NWORDS)

def correct(word):
  candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word] # 编辑距离为0的优先，>1,>2
  return max(candidates,key=lambda w:NWORDS[w])