梯度下降算法笔记

　　今天课上主要讲的是梯度下降算法。

　　上一次老师留了梯度下降算法求解线性回归的作业。大部分用java和C++实现的。

　　笔记也主要来自课程ppt，老师课程的ppt也主要参考的斯坦福吴恩达老师的讲义。

　　梯度下降法（英语：Gradient descent）是一个一阶最优化算法，通常也称为最速下降法。 要使用梯度下降法找到一个函数的局部极小值，必须向函数上当前点对应梯度（或者是近似梯度）的反方向的规定步长距离点进行迭代搜索。如果相反地向梯度正方向迭代进行搜索，则会接近函数的局部极大值点；这个过程则被称为梯度上升法。[来自维基百科]

　　

__author__ = 'Oscar_Yang'
#-*- coding= utf-8 -*-
"""
实现线性回归参数预测
"""
class GD(object):
    def __init__(self, e=0.03,m=5,rate=0.02,maxIterations=10000):#初始化属性
        self.e = e
        self.m = m
        self.rate = rate
        self.maxIterations=maxIterations
    """
    梯度下降拟合函数
    """
    def fit(self,X,y,data):
        self.theta = np.ones(data.shape[1])
        self.cost_ = []
        for i in range(0, self.maxIterations):
            output = self.predict(X)
            errors = (y - output)
            cost = (errors ** 2).sum() / 2.0
            self.cost_.append(cost)
            if cost < self.e:
                break
            else:
                gradient = np.dot(X.T,errors) / self.m
                self.theta = self.theta + self.rate * gradient
        return self
    """
        normal equation
    """
    @staticmethod
    def fit_equation(X,y):
        xTx = X.T.dot(X)
        XtX = np.linalg.inv(xTx)
        XtX_xT = XtX.dot(X.T)
        theta = XtX_xT.dot(y)
        return theta


    def predict(self, X):
        return np.dot(X,self.theta)
    @classmethod
    def cal_time(self):
        return time.clock()
if __name__ == '__main__':

    import pandas as pd
    import numpy as np
    import time
    import matplotlib.pyplot as plt

    data = pd.read_table("shuju1.txt")
    traindata0 = pd.DataFrame(np.ones(data.shape[0]))
    traindata1 = data.ix[:, 0:-1]
    traindata = pd.concat([traindata0, traindata1], axis=1)
    traindata_label = data.ix[:, -1]
    """
        normal equation
    """
    real_theta = GD(m=5,rate=0.01).fit_equation(traindata,traindata_label)
    print("精确参数：",real_theta)
    print("_____________________________________")
    fig,ax=plt.subplots(nrows=1,ncols=3,figsize=(18,6))
    start_time = GD.cal_time()
    ada1 = GD(m=5,rate=0.01).fit(traindata, traindata_label, data=data)
    end_time = GD.cal_time()
    during_time=end_time-start_time

    print("rate={}时候的参数为：{}".format(ada1.rate,ada1.theta))
    print("迭代次数：",len(ada1.cost_))
    print("耗时：",during_time)
    ax[0].plot(range(1,len(ada1.cost_)+1),np.log10(ada1.cost_),marker="o")
    ax[0].set_xlabel("Epochs")
    ax[0].set_ylabel('log(sum_squ_err)')
    title = "rate:", ada1.rate
    ax[0].set_title(title)

    start_time=GD.cal_time()
    ada2 = GD(rate=0.03,m=5).fit(traindata, traindata_label, data=data)
    end_time = GD.cal_time()
    print("_____________________________________")
    print("rate={}时候的参数为：{}".format(ada2.rate, ada2.theta))
    print("迭代次数：",len(ada2.cost_))
    print("耗时：",end_time-start_time)

    ax[1].plot(range(1, len(ada2.cost_)+1), np.log10(ada2.cost_), marker="o")
    ax[1].set_xlabel("Epochs")
    ax[1].set_ylabel('log(sum_squ_err)')
    title="rate:",ada2.rate
    ax[1].set_title(title)



    start_time=GD.cal_time()
    ada3 = GD( rate=0.06, m=5).fit(traindata, traindata_label, data=data)
    end_time=GD.cal_time()
    # print(ada3.theta)
    print("_____________________________________")
    print("rate={}时候的参数为：{}".format(ada3.rate, ada3.theta))
    print("迭代次数：",len(ada3.cost_))
    print("耗时：",end_time-start_time)

    ax[2].plot(range(1, len(ada3.cost_) + 1), np.log10(ada3.cost_), marker="o")
    ax[2].set_xlabel("Epochs")
    ax[2].set_ylabel('log(sum_squ_err)')
    title = "rate:", ada3.rate
    ax[2].set_title(title)

    plt.show()

精确参数： [ 1.75789474 1.21684211 -0.50526316]
_____________________________________
rate=0.01时候的参数为：[ 1.75789374 1.21684043 -0.50526106]
迭代次数： 10000
耗时： 7.593358084384351
_____________________________________
rate=0.03时候的参数为：[ 1.75789474 1.21684211 -0.50526316]
迭代次数： 10000
耗时： 7.8287073625201655
_____________________________________
rate=0.06时候的参数为：[ 1.75789474 1.21684211 -0.50526316]
迭代次数： 10000
耗时： 7.562434044109619