[模型评估]常用功能实现

前言：机器学习的主要任务是使用数据训练出模型，使用训练好的模型完成相关如分类、聚类、推荐等任务。

对于某类问题，可以建立多种不同的模型，也有多种评价指标对模型进行评估。

本文汇总了最常用的几种评价指标，方便快速查询使用。（本文使用鸢尾花数据集和逻辑回归模型）

1.常用评估指标及损失函数：

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import sklearn

from sklearn import datasets

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split

from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix

from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import r2_score


#1. 加载数据
iris = datasets.load_iris()
x,y = iris.data,iris.target

#2. 划分训练集与测试集
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=0)

#3. 使用逻辑回归训练分类器
lr_clf = LogisticRegression(C=1.0, penalty='l1', tol=1e-6, solver='liblinear',multi_class='ovr')
lr_clf.fit(x_train, y_train)

#4. 对测试集进行预测(分类)
y_predict = lr_clf.predict(x_test)
y_probs = lr_clf.predict_proba(x_test) #模型的预测得分

#5. 衡量指标
# #交叉验证
# cross_score = cross_val_score(lr_clf, x_train, y_train, cv=3, scoring="accuracy")
# print(cross_score)

#5.1 准确率——accuracy
accuracy = accuracy_score(y_test, y_predict)
print(accuracy)
print()

#5.2 精确率——precision
accuracy = precision_score(y_test, y_predict, average='macro')
print(accuracy)
print()

#5.3 召回率——recall
accuracy = recall_score(y_test, y_predict, average='weighted')
print(accuracy)
print()

#5.4 f1分数——f1_score
accuracy = f1_score(y_test, y_predict, average='micro')
print(accuracy)
print()

#5.5 混淆矩阵
confusion_matrix = confusion_matrix(y_test, y_predict)
print(confusion_matrix)
print()

#5.6 均方误差(MSE)
mse = mean_squared_error(y_test,y_predict)
print(mse)

#5.7 均方对数误差(MSLE)
msle = mean_squared_log_error(y_test,y_predict)
print(msle)

#5.8 平均绝对值误差(MAE)
mae = mean_absolute_error(y_test,y_predict)
print(mae)

#5.9 R2决定系数()
r2 = r2_score(y_test,y_predict)
print(r2)

2.ROC曲线与P-R曲线：二分类问题使用

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import sklearn

from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn import datasets
from sklearn.metrics import auc
from sklearn.metrics import roc_curve
from sklearn.metrics import precision_recall_curve

#1. 加载数据
iris = datasets.load_iris()
x,y = iris.data,iris.target

#2. 划分训练集与测试集
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=0)

# 进行二分类：0类和非0类
y_train = (y_train == 0)
y_test = (y_test == 0)

#3. 使用逻辑回归训练分类器
lr_clf = LogisticRegression(C=1.0, penalty='l1', tol=1e-6, solver='liblinear',multi_class='ovr')
lr_clf.fit(x_train, y_train)

#4. 对测试集进行预测(分类)
y_predict = lr_clf.predict(x_test)
y_probs = lr_clf.predict_proba(x_test) #模型的预测得分

cross_score = cross_val_score(lr_clf, x_train, y_train, cv=3, scoring="accuracy")
y_scores = cross_val_predict(lr_clf, x_train, y_train, cv=3,method="decision_function")


#5. 不同阈值下精度与召回率
precisions,recalls,thresholds = precision_recall_curve(y_train,y_scores)
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "r-", label="Precision", linewidth=1)
    plt.plot(thresholds, recalls[:-1], "b-", label="Recall", linewidth=1)
    plt.xlabel("Threshold", fontsize=12)
    plt.legend(loc="upper left", fontsize=12)
    plt.ylim([0, 1])

plt.figure(figsize=(8, 6))
plot_precision_recall_vs_threshold(precisions,recalls,thresholds)
plt.xlim([0, 6])
plt.show()


#6. P-R曲线
def plot_precision_vs_recall(precisions, recalls):
    plt.plot(recalls, precisions, "b-", linewidth=2)
    plt.xlabel("Recall", fontsize=16)
    plt.ylabel("Precision", fontsize=16)
    plt.axis([0, 1, 0, 1])

plt.figure(figsize=(8, 6))
plot_precision_vs_recall(precisions, recalls)
plt.show()


#7. ROC曲线
fpr, tpr, thresholds = roc_curve(y_train, y_scores)
def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, 'r--',linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.axis([0, 1, 0, 1])
    plt.xlabel('False Positive Rate', fontsize=16)
    plt.ylabel('True Positive Rate', fontsize=16)

plt.figure(figsize=(8, 6))
plot_roc_curve(fpr, tpr)
plt.show()

3.常用距离计算：余弦距离 = 1 - 余弦相似度，欧氏距离

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import sklearn

from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.metrics.pairwise import euclidean_distances

#1. 余弦相似度
a = [[1,3,2],[2,2,1]]
dist = cosine_similarity(a)
print(dist)
print()


#2. 余弦距离= 1 - 余弦相似度
dist = pairwise_distances(a,metric="cosine")
print(dist)
print()

#3. 欧氏距离
dist = euclidean_distances(a)
print(dist)
print()

4.数据集划分：holdout，cross-validation，bootstrap

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import sklearn

from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

from sklearn.model_selection import KFold
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import ShuffleSplit

from sklearn.model_selection import StratifiedKFold

iris = datasets.load_iris()
x,y = iris.data,iris.target

#1. holdout方式：用于划分训练集与测试集
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=0)


#2. 交叉验证：用于划分训练集和验证集

##2.1 K折交叉
# kf = KFold(n_splits=2)
# for train_idx, valid_idx in kf.split(x_train):
#     train_data = x_train[train_idx]
#     valid_data = x_train[valid_idx]
#     print("%s %s" % (train_data, valid_data))
# print()

##2.2 留一法
# loo = LeaveOneOut()
# for train_idx, valid_idx in loo.split(x_train):
#     train_data = x_train[train_idx]
#     valid_data = x_train[valid_idx]
#     print("%s %s" % (train_data, valid_data))
# print()

##2.3 随机洗牌
# ss = ShuffleSplit(n_splits=5, test_size=0.25,random_state=0)
# for train_idx, valid_idx in ss.split(x_train):
#     train_data = x_train[train_idx]
#     valid_data = x_train[valid_idx]
#     print("%s %s" % (train_data, valid_data))
# print()

##2.4 分层K折交叉
# skf = StratifiedKFold(n_splits=3)
# for train_idx, valid_idx in skf.split(x_train,y_train):
#     train_data = x_train[train_idx]
#     valid_data = x_train[valid_idx]
#     print("%s %s" % (train_data, valid_data))
# print()


#3. boostrap方式(自助法)：用于划分训练集与测试集
df_x = pd.DataFrame(x)
df_y = pd.DataFrame(y)
data = pd.concat([df_x,df_y],axis=1)
train = data.sample(frac=1.0,replace=True)
test = data.loc[data.index.difference(train.index)].copy()
#train中有m个样本，但是有重复的
print(len(train))

#test中有大约(m * 0.368)个样本，不重复，并且都未在train中出现过
print(len(test))

5.超参数调优：网格搜索，随机搜索

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import sklearn
from scipy.stats import randint
from sklearn import datasets
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV

iris = datasets.load_iris()
x,y = iris.data,iris.target
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=0)

knn = KNeighborsClassifier(n_neighbors=5)

#本题使用KNN分类模型，通过网格搜索在1～30之间选择超参数n_neighbors，获得参数及评分

##1. 网格搜索
# k_range = range(10, 15)
# param_grid = dict(n_neighbors=k_range)
# #param_grid = [{'n_neighbors':k_range},{'algorithm':['ball_tree','kd_tree'],'leaf_size':range(29,31),'n_neighbors':k_range}]
# grid = GridSearchCV(estimator=knn, param_grid=param_grid, cv=10, scoring='accuracy')
# grid.fit(x, y)


##2. 随机搜索
param_grid = {'n_neighbors':randint(low=1,high=31)}
grid = RandomizedSearchCV(estimator=knn, param_distributions=param_grid, cv=10, scoring='accuracy')
grid.fit(x, y)


#最佳参数选择
print(grid.best_params_)

#最佳得分
print(grid.best_score_)

#最佳评估器，可直接用于实际训练
best_knn = grid.best_estimator_
best_knn.fit(x_train, y_train)

pred_train = best_knn.predict(x_train)
pred_test = best_knn.predict(x_test)

train_acc = accuracy_score(y_train, pred_train)
test_acc = accuracy_score(y_test, pred_test)

#验证集与测试集得分
print(train_acc,test_acc)