liblinear使用总结

liblinear是libsvm的线性核的改进版本，专门适用于百万数据量的分类。正好适用于我这次数据挖掘的实验。

liblinear用法和libsvm很相似，我是用的是.exe文件，利用python的subprocess向控制台发送命令即可完成本次试验。

其中核心两句即

train train.txt

predict test.txt train.txt.model output.txt

由于是线性核，没有设置参数c、g

对于50W篇文章模型训练仅需340秒，50W篇文章的预测仅需6秒

from subprocess import *
import time

time = time.time

start_time = time()
print("训练")
cmd = "train train.txt"
Popen(cmd, shell = True, stdout = PIPE).communicate()
print("训练结束",str(time() - start_time))


start_time = time()
print("预测")
cmd = "predict test.txt train.txt.model output.txt"
Popen(cmd, shell = True).communicate()
print("预测结束",str(time() - start_time))


#进行统计
#读测试集真实label
start_time = time()
print("统计")
test_filename = "test.txt"
f = open(test_filename,"r",encoding = "utf-8")
real_class = []
for line in f:
    real_class.append(line[0])

#总样本
total_sample = len(real_class)

#读预测结果label
predict_filename = "output.txt"
f_predict = open(predict_filename,"r",encoding = "utf-8")
s = f_predict.read()
predict_class = s.split()

#对预测正确的文章进行计数
T = 0
for real, predict in zip(real_class,predict_class):
    if int(real) == int(predict):
        T += 1
accuracy  = T / total_sample * 100
print("正确率 为", str(accuracy) + "%")


# class_label = ["0","1","2","3","4","5","6","7","8","9"]
num_to_cate = {0:"it",1:"体育",2:"军事",3:"金融",4:"健康",5:"汽车",6:"房产",7:"文化",8:"教育",9:"娱乐"}

class_label = ["it","体育","军事","金融","健康","汽车","房产","文化","教育","娱乐"]

predict_precision = dict.fromkeys(class_label,1.0)
predict_true = dict.fromkeys(class_label,1.0)

predict_recall = dict.fromkeys(class_label,1.0)
predict_F = dict.fromkeys(class_label,0.0)
# print(str(predict_precision))
# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))
mat = dict.fromkeys(class_label,{})
for k,v in mat.items():
    mat[k] = dict.fromkeys(class_label,0)

# print(str(mat))

for real, predict in zip(real_class,predict_class):
    real = int(real)
    predict = int(predict)
    # print(num_to_cate[real])
    # print(num_to_cate[predict])
    mat[num_to_cate[real]][num_to_cate[predict]] += 1
    predict_precision[num_to_cate[predict]] += 1
    predict_recall[num_to_cate[real]] += 1

    if int(real) == int(predict):
        predict_true[num_to_cate[predict]] += 1

# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))

#输出混淆矩阵
for k, v in mat.items():
    print(k + ":" + str(v))

#计算精确率和召回率
for x in range(len(class_label)):
    # x =  str(x)
    predict_precision[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_precision[num_to_cate[x]]
    predict_recall[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_recall[num_to_cate[x]]

# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))

#计算F测度
for x in range(len(class_label)):
    # x = str(x)
    predict_F[num_to_cate[x]] = 2 * predict_recall[num_to_cate[x]] * predict_precision[num_to_cate[x]] / (predict_precision[num_to_cate[x]] + predict_recall[num_to_cate[x]])

print("统计结束",str(time() - start_time))
print("精确率为",str(predict_precision))
print("召回率为",str(predict_recall))
print("F测度为",str(predict_F))

print("保存结果")
final_result_filename = "./finalresult.txt"
f = open(final_result_filename,"w",encoding = "utf-8")
for k, v in mat.items():
    f.write(k + ":" + str(v) + "
")

f.write("
")
f.write("正确率为" + str(accuracy) + "%" + "

")
f.write("精确率为" + str(predict_precision) + "

")
f.write("召回率为" + str(predict_recall) + "

")
f.write("F测度为" + str(predict_F) + "

")
print("保存结果结束")


# cate_to_num = {"it":0,"体育":1,"军事":2,"华人":3,"国内":4,"国际":5,"房产":6,"文娱":7,"社会":8,"财经":9}
# num_to_cate = {0:"it",1:"体育",2:"军事",3:"华人",4:"国内",5:"国际",6:"房产",7:"文娱",8:"社会",9:"财经"}