《机器学习实战》笔记——树回归

线性回归的缺陷：

　　创建模型是需要拟合所有的样本（除了局部加权线性回归），当数据特征多且关系复杂时，显得太笨拙

树回归：

　　将数据集分成多分易建模的数据，然后在这些易于建模的小数据集上利用线性回归建模。树回归采用的是二元划分法，所以只可能产生二叉树。

CART算法

　　全称：classification and regression trees，分类回归树。在每个叶节点上使用各自的均值做预测，每个叶节点上包含单个值。建树期间，为了防止树的过拟合，需要使用到树剪枝技术。

模型树算法：

　　在每个叶节点上都构建出一个线性模型，即每个叶节点上包含一个线性方程。建树期间，需要调参，所以还回介绍Python中的Tkinter模块建立GUI。

# _*_ coding:utf-8 _*_

# 9-1 CART算法的代码实现
from numpy import *

def loadDataSet(fileName):
    dataMat = []
    fr = open(fileName)
    for line in fr.readlines():
        curLine = line.strip().split('	')
        fltLine = map(float, curLine)  # 每行映射成浮点数
        dataMat.append(fltLine)
    return dataMat

# 大于value的数据集划分到左树，小于等于value的数据集划分到右树
def binSplitDataSet(dataSet, feature, value):
    mat0 = dataSet[nonzero(dataSet[:, feature] > value)[0], :]
    mat1 = dataSet[nonzero(dataSet[:, feature] <= value)[0], :]
    return mat0, mat1


def regLeaf(dataSet):
    return mean(dataSet[:, -1])


def regErr(dataSet):
    return var(dataSet[:, -1]) * shape(dataSet)[0]



# 9-2 回归树的划分函数
# leafType是对创建叶节点的函数的引用
# errType是对总方差计算函数的引用
# ops是一个用户定义的参数构成的元祖，用于控制停止的时机，第一个值是容许的误差下降值，第二个是切分的最少样本数
def chooseBestSplit(dataSet, leafType=regLeaf, errType=regErr, ops=(1, 4)):
    tolS = ops[0]
    tolN = ops[1]
    if len(set(dataSet[:, -1].T.tolist()[0])) == 1:    # 勘误，书上多加了[0]
        return None, leafType(dataSet)
    m, n = shape(dataSet)
    S = errType(dataSet)
    bestS = inf
    bestIndex = 0
    bestValue = 0
    for featIndex in range(n - 1):
        for splitVal in set((dataSet[:, featIndex].T.A.tolist())[0]):
            mat0, mat1 = binSplitDataSet(dataSet, featIndex, splitVal)
            if (shape(mat0)[0] < tolN) or (shape(mat1)[0] < tolN): continue
            newS = errType(mat0) + errType(mat1)
            if newS < bestS:
                bestIndex = featIndex
                bestValue = splitVal
                bestS = newS
    if (S - bestS) < tolS:
        return None, leafType(dataSet)
    mat0, mat1 = binSplitDataSet(dataSet, bestIndex, bestValue)
    if (shape(mat0)[0] < tolN) or (shape(mat1)[0] < tolN):
        return None, leafType(dataSet)
    return bestIndex, bestValue  # 返回最好的特征和特征划分的依据值

# 建CART树
# step1 通过chooseBestSplit函数找到最佳划分特征feat以及划分用的标准val
# step2 若feat为空，证明已经到了叶子节点，返回val；若非空，则递归创建左右子树，返回存储左右子树的字典引用
def createTree(dataSet, leafType=regLeaf, errType=regErr,ops=(1, 4)):
    feat, val = chooseBestSplit(dataSet, leafType, errType, ops)
    if feat == None: return val
    retTree = {}
    retTree['spInd'] = feat
    retTree['spVal'] = val
    lSet, rSet = binSplitDataSet(dataSet, feat, val)
    retTree['left'] = createTree(lSet, leafType, errType, ops)
    retTree['right'] = createTree(rSet, leafType, errType, ops)
    return retTree


def isTree(obj):
    return (type(obj).__name__ == 'dict')


def getMean(tree):
    if isTree(tree['right']): tree['right'] = getMean(tree['right'])
    if isTree(tree['left']): tree['left'] = getMean(tree['left'])
    return (tree['left'] + tree['right']) / 2.0

# 后剪枝（建树之后进行）
# 准备工作：用训练集建树，构建的树要足够大，方便剪枝
# 步骤：如果左右树任意一棵是树，则在该子集递归剪枝过程；否则，计算并比较两个叶节点合并后的误差和不合并的误差，若误差变小，则合并
def prune(tree, testData):
    if shape(testData)[0] == 0: return getMean(tree)
    if (isTree(tree['right']) or isTree(tree['left'])):
        lSet, rSet = binSplitDataSet(testData, tree['spInd'], tree['spVal'])
    if isTree(tree['left']): tree['left'] = prune(tree['left'], lSet)
    if isTree(tree['right']): tree['right'] = prune(tree['right'], rSet)
    if not isTree(tree['left']) and not isTree(tree['right']):
        lSet, rSet = binSplitDataSet(testData, tree['spInd'], tree['spVal'])
        errorNoMerge = sum(power(lSet[:, -1] - tree['left'], 2)) + 
                       sum(power(rSet[:, -1] - tree['right'], 2))
        treeMean = (tree['left'] + tree['right']) / 2.0
        errorMerge = sum(power(testData[:, -1] - treeMean, 2))
        if errorMerge < errorNoMerge:
            print ("merging")
            return treeMean
        else:
            return tree
    else:
        return tree

# 9-4 模型树的叶节点生成函数
def linearSolve(dataSet):
    m,n = shape(dataSet)
    X = mat(ones((m,n)))
    Y = mat(ones((m,1)))
    X[:,1:n] = dataSet[:,0:n-1]
    Y = dataSet[:,-1]
    xTx = X.T * X
    if linalg.det(xTx) == 0.0:
        raise NameError('This matrix is singular, cannot do inverse,
try increasing the second value of ops')
        # 当程序出现错误，python会自动引发异常，也可以通过raise显示地引发异常。一旦执行了raise语句，raise后面的语句将不能执行。
    ws = xTx.I * (X.T * Y)
    return ws, X, Y


def modelLeaf(dataSet):
    ws, X, Y = linearSolve(dataSet)
    return ws


def modelErr(dataSet):
    ws, X, Y = linearSolve(dataSet)
    yHat = X * ws
    return sum(power(Y - yHat, 2))

myMat2 = mat(loadDataSet('exp2.txt'))
modelTrees = createTree(myMat2,modelLeaf,modelErr,(1,10))
print modelTrees