语料处理 合集

1.中文语料常常遇到编码问题，将任意字符集文件转为utf-8编码

import chardet 
import codecs
from django.utils.encoding import smart_text
 
def check_file_charset(file):                 #查看file文件的编码 
    with open(file, 'rb') as f:
        return chardet.detect(f.read())

def Convert_file_character(File_path):
    f_type = check_file_charset(File_path)
    print (File_path,"字符集为：",f_type['encoding'])       
    try:
        if f_type and 'encoding' in f_type.keys() and f_type['encoding'] != 'utf-8':
            with codecs.open(File_path, 'rb', f_type['encoding'],errors='ignore') as f:
                content = smart_text(f.read())
            with codecs.open(File_path, 'wb', 'utf-8') as f:
                f.write(content)
            print ("字符集转换成功")
        else:
            print("字符集为 utf-8，不需要进行转换")
    except Exception as ERR:
        print("字符集转换失败")
        print (ERR)

corpus_path = './unlabel'
raw_train_files = [corpus_path + os.sep + file_name for file_name in os.listdir(corpus_path)]
for raw_train_file in raw_train_files:
    Convert_file_character(raw_train_file)

参考：https://blog.csdn.net/qq_35751770/article/details/103664496

2.将unlabel文件夹中的所有.txt文件合并，每个文件之间空一行 

先调用上面的代码转换编码

def combine(corpus_path, outpath):
    output = open(outpath, 'a', encoding='utf-8')
    
    raw_train_files = [corpus_path + os.sep + file_name for file_name in os.listdir(corpus_path)]
    for raw_train_file in raw_train_files:
        
        f_type = check_file_charset(raw_train_file)                    #查看文件的编码 
        print (raw_train_file,"字符集为：",f_type['encoding'])
        with open(raw_train_file, 'r+', encoding='utf-8') as f:
            context = f.readlines()
    
        for x in context:
            output.write(x)
        output.write('
')

combine('./unlabel', 'all_unlabel.txt')

3.随机抽取.txt文件中的60%，20%，5%

def part(filename, outpath, ratio): 
    output = open(outpath, 'w+', encoding='utf-8')
    context = []
    with open(filename, 'r+', encoding='utf-8') as f:
        context.extend(f.readlines())
    
    length = len(context)
    random_order = list(range(length))
    np.random.shuffle(random_order)
    
    batch_size = int(length*ratio)
    print(batch_size)
    for x in context[:batch_size]:
        output.write(x)
        
ratio1, ratio2, ratio3 = 0.6, 0.2, 0.05
part('training/law_train.txt', 'training/law_train1.txt', ratio1)    
part('training/law_train.txt', 'training/law_train2.txt', ratio2)           
part('training/law_train.txt', 'training/law_train3.txt', ratio3) 

4.将已经分好词的文件去掉空格（正则），恢复成文件原来的样子

def deal_data(filename, outpath): 
    output = open(outpath, 'w+', encoding='utf-8')
    
    with open(filename, 'r+', encoding='utf-8') as f:
        context = f.readlines()
        for data in context:                 #data为某一行数据             
            x = re.sub('s+', '', data).strip()
            output.write(x)
            

deal_data('evaluate/law/Law_contract_test.txt', 'evaluate/gold/Law_contract_test.txt')
deal_data('evaluate/law/Law_marriage_test.txt', 'evaluate/gold/Law_marriage_test.txt')            
deal_data('evaluate/law/Law_mixed_test.txt', 'evaluate/gold/Law_mixed_test.txt')