鑷劧璇█澶勭悊锛氭枃鏈澶勭悊銆佽瑷€妯″瀷銆丷NN

Posted 2020-11-25

鏍囩锛?a href='http://www.mamicode.com/so/1/batch' title='batch'>batch   鎵归噺   浣嶇疆   shape   red   lin   char   etc   淇濈暀   

鏂囨湰鏄竴绫诲簭鍒楁暟鎹紝涓€绡囨枃绔犲彲浠ョ湅浣滄槸瀛楃鎴栧崟璇嶇殑搴忓垪锛屾湰鑺傚皢浠嬬粛鏂囨湰鏁版嵁鐨勫父瑙侀澶勭悊姝ラ锛岄澶勭悊閫氬父鍖呮嫭鍥涗釜姝ラ锛?/p>

1. 璇诲叆鏂囨湰
2. 鍒嗚瘝
3. 寤虹珛瀛楀吀锛屽皢姣忎釜璇嶆槧灏勫埌涓€涓敮涓€鐨勭储寮曪紙index锛?/li>
4. 灏嗘枃鏈粠璇嶇殑搴忓垪杞崲涓虹储寮曠殑搴忓垪锛屾柟渚胯緭鍏ユā鍨?/li>

鐜版湁鐨勫伐鍏峰彲浠ュ緢濂藉湴杩涜鍒嗚瘝锛?a href="https://spacy.io/" target="_blank">spaCy鍜?a href="https://www.nltk.org/" target="_blank">NLTK銆?/p>

 

浣跨敤绀轰緥锛?/p>

text = "Mr. Chen doesn鈥榯 agree with my suggestion."
####spaCy:
import spacy
nlp = spacy.load(鈥?/span>en_core_web_sm鈥?/span>)
doc = nlp(text)
print([token.text for token in doc])
[鈥?/span>Mr.鈥?/span>, 鈥?/span>Chen鈥?/span>, 鈥?/span>does鈥?/span>, "n鈥榯", 鈥?/span>agree鈥?/span>, 鈥?/span>with鈥?/span>, 鈥?/span>my鈥?/span>, 鈥?/span>suggestion鈥?/span>, 鈥?/span>.鈥?/span>]




####NLTK:
from nltk.tokenize import word_tokenize
from nltk import data
data.path.append(鈥?/span>/home/kesci/input/nltk_data3784/nltk_data鈥?/span>)
print(word_tokenize(text))
[鈥?/span>Mr.鈥?/span>, 鈥?/span>Chen鈥?/span>, 鈥?/span>does鈥?/span>, "n鈥榯", 鈥?/span>agree鈥?/span>, 鈥?/span>with鈥?/span>, 鈥?/span>my鈥?/span>, 鈥?/span>suggestion鈥?/span>, 鈥?/span>.鈥?/span>]

璇█妯″瀷

涓€娈佃嚜鐒惰瑷€鏂囨湰鍙互鐪嬩綔鏄竴涓鏁ｆ椂闂村簭鍒楋紝缁欏畾涓€涓暱搴︿负TT鐨勮瘝鐨勫簭鍒?span class="MathJax_Preview">w1,w2,…,wTw1,w2,…,wT锛岃瑷€妯″瀷鐨勭洰鏍囧氨鏄瘎浼拌搴忓垪鏄惁鍚堢悊锛屽嵆璁＄畻璇ュ簭鍒楃殑姒傜巼锛?/span>

P(w1,w2,…,wT).P(w1,w2,…,wT).

N-gram璇█妯″瀷

涓轰簡瑙ｅ喅鑷敱鍙傛暟鏁扮洰杩囧鐨勯棶棰橈紝寮曞叆浜嗛┈灏旂澶亣璁撅細闅忔剰涓€涓瘝鍑虹幇鐨勬鐜囧彧涓庡畠鍓嶉潰鍑虹幇鐨勬湁闄愮殑n涓瘝鏈夊叧銆傚熀浜庝笂杩板亣璁剧殑缁熻璇█妯″瀷琚О涓篘-gram璇█妯″瀷銆?/span>

闅忔満閲囨牱

鍦ㄩ殢鏈洪噰鏍蜂腑锛屾瘡涓牱鏈槸鍘熷搴忓垪涓婁换鎰忔埅鍙栫殑涓€娈靛簭鍒楋紝鐩搁偦鐨勪袱涓殢鏈哄皬鎵归噺鍦ㄥ師濮嬪簭鍒椾笂鐨勪綅缃笉涓€瀹氱浉姣楅偦銆?/p>

import torch
import random
def data_iter_random(corpus_indices, batch_size, num_steps, device=None):
    # 鍑?鏄洜涓哄浜庨暱搴︿负n鐨勫簭鍒楋紝X鏈€澶氬彧鏈夊寘鍚叾涓殑鍓峮 - 1涓瓧绗?/span>
    num_examples = (len(corpus_indices) - 1) // num_steps  # 涓嬪彇鏁达紝寰楀埌涓嶉噸鍙犳儏鍐典笅鐨勬牱鏈釜鏁?/span>
    example_indices = [i * num_steps for i in range(num_examples)]  # 姣忎釜鏍锋湰鐨勭涓€涓瓧绗﹀湪corpus_indices涓殑涓嬫爣
    random.shuffle(example_indices)

    def _data(i):
        # 杩斿洖浠巌寮€濮嬬殑闀夸负num_steps鐨勫簭鍒?/span>
        return corpus_indices[i: i + num_steps]
    if device is None:
        device = torch.device(鈥?/span>cuda鈥?/span> if torch.cuda.is_available() else 鈥?/span>cpu鈥?/span>)
    
    for i in range(0, num_examples, batch_size):
        # 姣忔閫夊嚭batch_size涓殢鏈烘牱鏈?/span>
        batch_indices = example_indices[i: i + batch_size]  # 褰撳墠batch鐨勫悇涓牱鏈殑棣栧瓧绗︾殑涓嬫爣
        X = [_data(j) for j in batch_indices]
        Y = [_data(j + 1) for j in batch_indices]
        yield torch.tensor(X, device=device), torch.tensor(Y, device=device)

鐩搁偦閲囨牱

鍦ㄧ浉閭婚噰鏍蜂腑锛岀浉閭荤殑涓や釜闅忔満灏忔壒閲忓湪鍘熷搴忓垪涓婄殑浣嶇疆鐩告瘲閭汇€?/p>

def data_iter_consecutive(corpus_indices, batch_size, num_steps, device=None):
    if device is None:
        device = torch.device(鈥?/span>cuda鈥?/span> if torch.cuda.is_available() else 鈥?/span>cpu鈥?/span>)
    corpus_len = len(corpus_indices) // batch_size * batch_size  # 淇濈暀涓嬫潵鐨勫簭鍒楃殑闀垮害
    corpus_indices = corpus_indices[: corpus_len]  # 浠呬繚鐣欏墠corpus_len涓瓧绗?/span>
    indices = torch.tensor(corpus_indices, device=device)
    indices = indices.view(batch_size, -1)  # resize鎴?batch_size, )
    batch_num = (indices.shape[1] - 1) // num_steps
    for i in range(batch_num):
        i = i * num_steps
        X = indices[:, i: i + num_steps]
        Y = indices[:, i + 1: i + num_steps + 1]
        yield X, Y

 

 寰幆绁炵粡缃戠粶寮曞叆涓€涓殣钘忓彉閲?span class="MathJax_Preview">HH锛岀敤HtHt琛ㄧずHH鍦ㄦ椂闂存tt鐨勫€笺€?span class="MathJax_Preview">HtHt鐨勮绠楀熀浜?span class="MathJax_Preview">XtXt鍜?span class="MathJax_Preview">Ht−1Ht−1锛屽彲浠ヨ涓?span class="MathJax_Preview">HtHt璁板綍浜嗗埌褰撳墠瀛楃涓烘鐨勫簭鍒椾俊鎭紝鍒╃敤HtHt瀵瑰簭鍒楃殑涓嬩竴涓瓧绗﹁繘琛岄娴嬨€?/span>

def rnn(inputs, state, params):
    # inputs鍜宱utputs鐨嗕负num_steps涓舰鐘朵负(batch_size, vocab_size)鐨勭煩闃?/span>
    W_xh, W_hh, b_h, W_hq, b_q = params
    H, = state
    outputs = []
    for X in inputs:
        H = torch.tanh(torch.matmul(X, W_xh) + torch.matmul(H, W_hh) + b_h)
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H,)

瑙ｅ喅寰幆绁炵粡缃戠粶姊害鐖嗙偢闂锛氭搴﹁鍓?/p>

瑁佸壀姊害锛坈lip gradient锛夋槸涓€绉嶅簲瀵规搴︾垎鐐哥殑鏂规硶銆傚亣璁炬垜浠妸鎵€鏈夋ā鍨嬪弬鏁扮殑姊害鎷兼帴鎴愪竴涓悜閲?nbsp;g锛屽苟璁捐鍓殑闃堝€兼槸θ銆傝鍓悗鐨勬搴?/span>

 

瑁佸壀姊害浠ｇ爜锛?/p>

def grad_clipping(params, theta, device):
    norm = torch.tensor([0.0], device=device)
    for param in params:
        norm += (param.grad.data ** 2).sum()
    norm = norm.sqrt().item()
    if norm > theta:
        for param in params:
            param.grad.data *= (theta / norm)

浣跨敤RNN妯″瀷杩涜棰勬祴锛?/p>

def predict_rnn(prefix, num_chars, rnn, params, init_rnn_state,
                num_hiddens, vocab_size, device, idx_to_char, char_to_idx):
    state = init_rnn_state(1, num_hiddens, device)
    output = [char_to_idx[prefix[0]]]   # output璁板綍prefix鍔犱笂棰勬祴鐨刵um_chars涓瓧绗?/span>
    for t in range(num_chars + len(prefix) - 1):
        # 灏嗕笂涓€鏃堕棿姝ョ殑杈撳嚭浣滀负褰撳墠鏃堕棿姝ョ殑杈撳叆
        X = to_onehot(torch.tensor([[output[-1]]], device=device), vocab_size)
        # 璁＄畻杈撳嚭鍜屾洿鏂伴殣钘忕姸鎬?/span>
        (Y, state) = rnn(X, state, params)
        # 涓嬩竴涓椂闂存鐨勮緭鍏ユ槸prefix閲岀殑瀛楃鎴栬€呭綋鍓嶇殑鏈€浣抽娴嬪瓧绗?/span>
        if t < len(prefix) - 1:
            output.append(char_to_idx[prefix[t + 1]])
        else:
            output.append(Y[0].argmax(dim=1).item())
    return 鈥樷€?/span>.join([idx_to_char[i] for i in output])