HMM实现中文分词

Posted 2022-07-31 刘润森！

import numpy as np
import warnings
from hmmlearn.hmm import MultinomialHMM as mhmm
data=[
 u"我要吃饭":"SSBE",

u"天气不错" : "BEBE",

u"谢天谢地" : "BMME"]
def prints(s):
    pass
    print(s)
def get_startprob():
    """get BMES matrix   
    """
    c=0
    c_map="B":0,"M":0,"E":0,"S":0
    #caculate the count
    for v in data :
        for key in v :
            value=v[key]
        c=c+1
        prints("value[0] is "+value[0])
        c_map[value[0]]=c_map[value[0]] +1
        prints("c_map[value[0]] is "+str(c_map[value[0]]) )
    res=[]
    for i in "BMES":
         res.append( c_map[i] / float(c))
    return res

def get_transmat():
    """get transmat of status
    """
    c=0
    #record BE:1,BB:2
    c_map=
    for v in data :
        for key in v :
            value=v[key]
        
        prints("value[0] is "+value[0])
        for v_i in range(len(value)-1):
            couple=value[v_i:v_i+2]
            c_couple_source = c_map.get(couple,0)
            c_map[couple]=c_couple_source+1
            c=c+1         
        #c_map[value[0]]=c_map[value[0]] +1
        #prints("c_map[value[0]] is "+str(c_map[value[0]]) )
    prints("get_transmat's c_map is "+str(c_map))
    res=[]
    for i in "BMES":
         col=[]
         col_count=0
         for j in "BMES":
             col_count=c_map.get(i+j,0)+col_count          
         for j in "BMES":      
             col.append( c_map.get(i+j,0) / float(col_count))
         res.append(col)
    return res
def get_words():
    return u"我要吃饭天气不错谢天地"
def get_word_map():
    words=get_words()
    res=
    for i in range(len(words)):
        res[words[i]]=i
    return res
def get_array_from_phase(phase):
    word_map=get_word_map()
    res=[]
    for key in phase:
        res.append(word_map[key])
    return res
def get_emissionprob():
    #get emmissionprob of status and observers
    c=0
    #record Bc=0
    #record B我:1,B吃:2
    c_map=
    for v in data :      
        for key in v :
            k=key
            value=v[key]
        prints("value[0] is "+value[0])
        for v_i in range(len(value)):
            couple=value[v_i]+k[v_i]
            prints("emmition's couple is " + couple)
            c_couple_source = c_map.get(couple,0)
            c_map[couple]=c_couple_source+1
            c=c+1
    res=[]
    prints("emmition's c_map is "+str(c_map))
    words=get_words()
    for i in "BMES":
         col=[]
         for j in words:
             col.append( c_map.get(i+j,0) / float(c))
         res.append(col)
    return res
if( __name__ == "__main__"):
    # print("startprob is ",get_startprob())
    # print("transmat is " ,get_transmat())
    print("emissionprob is " , get_emissionprob())
    print("word map is ",get_word_map())
    # coding=utf-8
    warnings.filterwarnings("ignore")
    # import matplotlib.pyplot as plt
    startprob = np.array(get_startprob())
    print("startprob is ", startprob)
    transmat = np.array(get_transmat())
    print("transmat is ", transmat)
    emissionprob = np.array(get_emissionprob())
    print("emmissionprob is ", emissionprob)
    mul_hmm = mhmm(n_components=4)
    mul_hmm.startprob_ = startprob
    mul_hmm.transmat_ = transmat
    mul_hmm.emissionprob_ = emissionprob
    phase = u"我要吃饭谢天谢地"
    X = np.array(get_array_from_phase(phase))
    X = X.reshape(len(phase), 1)
    print("X is ", X)
    Y = mul_hmm.predict(X)
    print("Y is ", Y)
    # B（词开头），M（词中），E（词尾），S（独字词） 0,1,2,3

out

F:\\anaconda\\pythonw.exe D:/学习资料/网易云课堂/唐宇迪-机器学习课程(新)/自然语言处理(Python版)/第八章：HMM实战/HMM案例实战/HMM/get_hmm_param.py
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is 'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1
emissionprob is  [[0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.0, 0.0, 0.08333333333333333, 0.08333333333333333, 0.0], [0.0, 0.0, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.08333333333333333, 0.0, 0.0, 0.08333333333333333], [0.08333333333333333, 0.08333333333333333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]
word map is  '我': 0, '要': 1, '吃': 2, '饭': 3, '天': 9, '气': 5, '不': 6, '错': 7, '谢': 8, '地': 10
value[0] is S
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 1
value[0] is B
c_map[value[0]] is 2
startprob is  [0.66666667 0.         0.         0.33333333]
value[0] is S
value[0] is B
value[0] is B
get_transmat's c_map is 'SS': 1, 'SB': 1, 'BE': 3, 'EB': 1, 'BM': 1, 'MM': 1, 'ME': 1
transmat is  [[0.   0.25 0.75 0.  ]
 [0.   0.5  0.5  0.  ]
 [1.   0.   0.   0.  ]
 [0.5  0.   0.   0.5 ]]
value[0] is S
emmition's couple is S我
emmition's couple is S要
emmition's couple is B吃
emmition's couple is E饭
value[0] is B
emmition's couple is B天
emmition's couple is E气
emmition's couple is B不
emmition's couple is E错
value[0] is B
emmition's couple is B谢
emmition's couple is M天
emmition's couple is M谢
emmition's couple is E地
emmition's c_map is 'S我': 1, 'S要': 1, 'B吃': 1, 'E饭': 1, 'B天': 1, 'E气': 1, 'B不': 1, 'E错': 1, 'B谢': 1, 'M天': 1, 'M谢': 1, 'E地': 1
emmissionprob is  [[0.         0.         0.08333333 0.         0.08333333 0.
  0.08333333 0.         0.08333333 0.08333333 0.        ]
 [0.         0.         0.         0.         0.08333333 0.
  0.         0.         0.08333333 0.08333333 0.        ]
 [0.         0.         0.         0.08333333 0.         0.08333333
  0.         0.08333333 0.         0.         0.08333333]
 [0.08333333 0.08333333 0.         0.         0.         0.
  0.         0.         0.         0.         0.        ]]
X is  [[ 0]
 [ 1]
 [ 2]
 [ 3]
 [ 8]
 [ 9]
 [ 8]
 [10]]
Y is  [3 3 0 2 0 1 1 2]

Process finished with exit code 0