使用BERT词向量

Posted 2020-12-16 xuehuiping

1. 启动远程服务

pip install --ignore-installed --upgrade tensorflow==1.10
pip install bert-serving-server
pip install bert-serving-client

下载模型

mkdir model
cd model
wget https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip
#启动服务
bert-serving-start -model_dir chinese_L-12_H-768_A-12/ -num_worker=8 -max_seq_len=40

2. 使用BertClient

pip install bert-serving-server

from functools import reduce
import numpy as np
from bert_serving.client import BertClient


def normaliz_vec(vec_list):
    for i in range(len(vec_list)):
        vec = vec_list[i]
        square_sum = reduce(lambda x, y: x + y, map(lambda x: x * x, vec))
        sqrt_square_sum = np.sqrt(square_sum)
        coef = 1 / sqrt_square_sum
        vec = list(map(lambda x: x * coef, vec))
        vec_list[i] = vec
    return vec_list


bc = BertClient(ip=‘XXX‘)

data = ‘你 好 啊‘.split(‘ ‘)
vectors = bc.encode(data)
question_vectors = normaliz_vec(vectors.tolist())
print(question_vectors)