NLP讯飞英文学术论文分类挑战赛Top10开源多方案–4 机器学习LGB 方案

Posted 2021-09-05 Better Bench

1 相关信息

• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–1 赛后总结与分析
• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–2 数据分析
• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–3 TextCNN Fasttext 方案
• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–4 机器学习LGB 方案
• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–5 Bert 方案
• 【NLP】讯飞英文学术论文分类挑战赛Top10开源多方案–6 提分方案

2 引言

（1）机器学习方法在我们的任务中，没有重点研究，仅写出一个简单的baseline。该部分由队友博远实现，比赛最后的模型融合的部分，使用到，单个LGB模型，没有调参，5折交叉验证，线上都能达到0.79+的成绩，还有很多可提分点，如果时间允许，有很多提分技巧

• XGB模型
• LR模型
• 多模型融合EnsembleVoteClassifier
• 多模型融合StackingClassifier
• 加入伪标签训练
• 数据增强
  （2）实现步骤
• 数据预处理
• K折划分数据
• TF-IDF提取特征，将单词转为数值矩阵
• 训练模型
• 每折预测一遍结果求和，最后取K折的结果的平均作为预测矩阵
• np.argmax取得预测值
• 生成提交文件

3 实现

import pandas as pd
from nltk.stem import WordNetLemmatizer
import re
import nltk
from spacy.lang.en.stop_words import STOP_WORDS
import numpy as np
import lightgbm as lgb
from sklearn import metrics
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import KFold, StratifiedKFold

from sklearn.model_selection import train_test_split

clean_tag =True
if clean_tag ==True:
    en_stop = set(nltk.corpus.stopwords.words('english'))
    custom_stop_words = [
        'doi', 'preprint', 'copyright', 'peer', 'reviewed', 'org', 'https', 'et', 'al', 'author', 'figure',
        'rights', 'reserved', 'permission', 'used', 'using', 'biorxiv', 'medrxiv', 'license', 'fig', 'fig.',
        'al.', 'elsevier', 'pmc', 'czi', 'www'
    ]
    for word in custom_stop_words:
        en_stop.add(word)


    def preprocess_text(document):
        stemmer = WordNetLemmatizer()

        document = str(document)
        document = document.replace("\\n", ' ')
        document = document.replace("/'", '')
        # Remove  all the special characters
        document = re.sub(r'\\W', ' ', document)

        # 删除所有单个字符
        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)

        # 从开头删除单个字符
        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)

        # 用单个空格替换多个空格
        document = re.sub(r'\\s+', ' ', document, flags=re.I)

        # 数字泛化：，所有大于9的数字都被hashs替换了。即成为# #,123变成# # #或15.80€变成# #,# #€。
        document = re.sub('[0-9]{5,}', '#####', document)
        document = re.sub('[0-9]{4}', '####', document)
        document = re.sub('[0-9]{3}', '###', document)
        document = re.sub('[0-9]{2}', '##', document)
        # 转换为小写
        document = document.lower()
        # 词形还原
        tokens = document.split()
        tokens = [stemmer.lemmatize(word) for word in tokens]
        # 去停用词
        tokens = [word for word in tokens if word not in en_stop]
        # 去低频词
        tokens = [word for word in tokens if len(word) > 3]
        preprocessed_text = ' '.join(tokens)

        return preprocessed_text

    train = pd.read_csv("train/train_stop.csv", sep="\\t")
    test = pd.read_csv("test/test.csv", sep="\\t")
    sub = pd.read_csv("sample_submit.csv")

    train["text"] = train["title"] + " " + train["abstract"]
    # for i in range(len(train["text"])):
    #     train["text"][i] = preprocess_text(train["text"][i])
    train["text"] = train["text"].progress_apply(lambda x: preprocess_text(x))
    train.to_csv('ml_clean_data.csv', sep='\\t')
else:
    train = pd.read_csv('ml_clean_data.csv', sep='\\t')
# 建立映射
label_id2cate = dict(enumerate(train.categories.unique()))
label_cate2id = {value: key for key, value in label_id2cate.items()}
train["label"] = train["categories"].map(label_cate2id)
df = train[["text", "label"]]
df.head()

# 生成提交文件
def submit_file(result_pred,label_id2cate):#result_pred是预测的结果，应该是10000个值
    print("存储预测结果")
    sub=pd.read_csv('./sample_submit.csv')# 官网给出的格式文件
    sub['categories']=list(result_pred)
    sub['categories']=sub['categories'].map(label_id2cate)
    sub.to_csv('submit/submit_{}_ensemble.csv'.format(models_name), index=False)

# 5折交叉验证

params = {
    "device_type": "gpu",
    "max_depth": 5,
    "min_data_in_leaf": 20,
    "num_leaves": 35,
    "learning_rate": 0.1,
    "lambda_l1": 0.1,
    "lambda_l2": 0.2,
    "objective": "multiclass",
    "num_class": 39,
    "verbose": 0,
}

train_data = df["text"]
train_label = df["label"]

NFOLDS = 5
kfold = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=1)
kf = kfold.split(train_data, train_label)
cv_pred = np.zeros(test.shape[0])
valid_best = 0

for i, (train_fold, validate) in enumerate(kf):

    #     X=train_data.reset_index(drop=True)
    #     y= train_label.reset_index(drop=True)
    X_train, X_validate, label_train, label_validate = (
        train_data.iloc[train_fold],
        train_data.iloc[validate],
        train_label[train_fold],
        train_label[validate],
    )

    # 将语料转化为词袋向量，根据词袋向量统计TF-IDF
    vectorizer = CountVectorizer(max_features=50000)
    tf_idf_transformer = TfidfTransformer()
    tf_idf = tf_idf_transformer.fit_transform(vectorizer.fit_transform(X_train))
    X_train_weight = tf_idf.toarray()  # 训练集TF-IDF权重矩阵
    tf_idf = tf_idf_transformer.transform(vectorizer.transform(X_validate))
    X_validate_weight = tf_idf.toarray()  # 验证集TF-IDF权重矩阵

    dtrain = lgb.Dataset(X_train_weight, label_train)
    dvalid = lgb.Dataset(X_validate_weight, label_validate, reference=dtrain)

    bst = lgb.train(
        params,
        dtrain,
        num_boost_round=10000,
        valid_sets=dvalid,
        early_stopping_rounds=500,
    )

    preds_last = bst.predict(test, num_iteration=bst.best_iteration)
    cv_pred += bst.predict(test, num_iteration=bst.best_iteration)
    valid_best += bst.best_score["valid_0"]["auc"]

cv_pred /= NFOLDS  # 预测输出
valid_best /= NFOLDS
result =np.argmax(cv_pred,axis=1) 
submit_file(list(result),label_id2cate)