结巴分词和自然语言处理HanLP处理手记

Posted 2020-10-10

手记实用系列文章：

1 结巴分词和自然语言处理HanLP处理手记

2 Python中文语料批量预处理手记

3 自然语言处理手记

4 Python中调用自然语言处理工具HanLP手记

5 Python中结巴分词使用手记

代码封装类：

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import jieba
import os
import re
import time
from jpype import *

\'\'\'
title：利用结巴分词进行文本语料的批量处理
    1 首先对文本进行遍历查找
    2 创建原始文本的保存结构
    3 对原文本进行结巴分词和停用词处理
    4 对预处理结果进行标准化格式，并保存原文件结构路径
author：白宁超
myblog：http://www.cnblogs.com/baiboy/
time：2017年4月28日10:03:09
\'\'\'


\'\'\'
创建文件目录
path:根目录下创建子目录
\'\'\'
def mkdir(path):
    # 判断路径是否存在
    isExists=os.path.exists(path)
    # 判断结果
    if not isExists:
        os.makedirs(path)
        print(path+\' 创建成功\')
        return True
    else:
        pass
    print(\'-->请稍后，文本正在预处理中...\')


\'\'\'
结巴分词工具进行中文分词处理：
read_folder_path：待处理的原始语料根路径
write_folder_path 中文分词经数据清洗后的语料
\'\'\'
def CHSegment(read_folder_path,write_folder_path):
    stopwords ={}.fromkeys([line.strip() for line in open(\'../Database/stopwords/CH_stopWords.txt\',\'r\',encoding=\'utf-8\')]) # 停用词表
    # 获取待处理根目录下的所有类别
    folder_list = os.listdir(read_folder_path)
    # 类间循环
    # print(folder_list)
    for folder in folder_list:
        #某类下的路径
        new_folder_path = os.path.join(read_folder_path, folder)
        # 创建一致的保存文件路径
        mkdir(write_folder_path+folder)
         #某类下的保存路径
        save_folder_path = os.path.join(write_folder_path, folder)
        #某类下的全部文件集
        # 类内循环
        files = os.listdir(new_folder_path)
        j = 1
        for file in files:
            if j > len(files):
                break
            # 读取原始语料
            raw = open(os.path.join(new_folder_path, file),\'r\',encoding=\'utf-8\').read()
            # 只保留汉字
            # raw1 = re.sub("[A-Za-z0-9\\[\\`\\~\\!\\@\\#\\$\\^\\&\\*\\(\\)\\=\\|\\{\\}\\\'\\:\\;\\\'\\,\\[\\]\\.\\<\\>\\/\\?\\~\\！\\@\\#\\\\\\&\\*\\%]", "", raw)
            # jieba分词
            wordslist = jieba.cut(raw, cut_all=False) # 精确模式
            # 停用词处理
            cutwordlist=\'\'
            for word in wordslist:
                if word not in stopwords and word=="\\n":
                    cutwordlist+="\\n" # 保持原有文本换行格式
                elif len(word)>1 :
                        cutwordlist+=word+"/" #去除空格
            #保存清洗后的数据
            with open(os.path.join(save_folder_path,file),\'w\',encoding=\'utf-8\') as f:
                f.write(cutwordlist)
                j += 1


\'\'\'
结巴分词工具进行中文分词处理：
read_folder_path：待处理的原始语料根路径
write_folder_path 中文分词经数据清洗后的语料
\'\'\'
def HanLPSeg(read_folder_path,write_folder_path):
    startJVM(getDefaultJVMPath(), "-Djava.class.path=C:\\hanlp\\hanlp-1.3.2.jar;C:\\hanlp", "-Xms1g", "-Xmx1g") # 启动JVM，Linux需替换分号;为冒号:
    stopwords ={}.fromkeys([line.strip() for line in open(\'../Database/stopwords/CH_stopWords.txt\',\'r\',encoding=\'utf-8\')]) # 停用词表
    # 获取待处理根目录下的所有类别
    folder_list = os.listdir(read_folder_path)
    # 类间循环
    # print(folder_list)
    for folder in folder_list:
        #某类下的路径
        new_folder_path = os.path.join(read_folder_path, folder)
        # 创建一致的保存文件路径
        mkdir(write_folder_path+folder)
         #某类下的保存路径
        save_folder_path = os.path.join(write_folder_path, folder)
        #某类下的全部文件集
        # 类内循环
        files = os.listdir(new_folder_path)
        j = 1
        for file in files:
            if j > len(files):
                break
            # 读取原始语料
            raw = open(os.path.join(new_folder_path, file),\'r\',encoding=\'utf-8\').read()
            # HanLP分词
            HanLP = JClass(\'com.hankcs.hanlp.HanLP\')
            wordslist = HanLP.segment(raw)
            #保存清洗后的数据
            wordslist1=str(wordslist).split(",")
            # print(wordslist1[1:len(wordslist1)-1])

            flagresult=""
            # 去除标签
            for v in wordslist1[1:len(wordslist1)-1]:
                if "/" in v:
                    slope=v.index("/")
                    letter=v[1:slope]
                    if len(letter)>0 and \'\\n\\u3000\\u3000\' in letter:
                        flagresult+="\\n"
                    else:flagresult+=letter +"/" #去除空格
            # print(flagresult)
            with open(os.path.join(save_folder_path,file),\'w\',encoding=\'utf-8\') as f:
                f.write(flagresult.replace(\' /\',\'\'))
            j += 1
    shutdownJVM()

if __name__ == \'__main__\' :
    print(\'开始进行文本分词操作：\\n\')
    t1 = time.time()

    dealpath="../Database/SogouC/FileTest/"
    savepath="../Database/SogouCCut/FileTest/"

    # 待分词的语料类别集根目录
    read_folder_path = \'../Database/SogouC/FileNews/\'
    write_folder_path = \'../Database/SogouCCut/\'

    #jieba中文分词
    CHSegment(read_folder_path,write_folder_path) #300个txtq其中结巴分词使用3.31秒
    HanLPSeg(read_folder_path,write_folder_path) #300个txt其中hanlp分词使用1.83秒

    t2 = time.time()
    print(\'完成中文文本切分: \'+str(t2-t1)+"秒。")

运行效果：