神经网络6：LSTM 神经网络

▶ 循环神经网络

● 代码，参考【https://zybuluo.com/hanbingtao/note/581764】，这里主要实现了一个单层 LSTM 神经网络类 LstmLayer，包含前向量和后向计算。

■ 原代码有多处错误，包括且不仅限于：Line 106/110/130: 传入的 activator 根本没用到；Line 254: delta_f_list -> delta_i_list；Line 256: delta_f_list -> delta_o_list；在 github 上反馈给了 Up 主，不知道有没有改善

■ 源代码有大量低效的循环和函数调用，在这里进行了数据结构调整和向量化。例如，类 LstmLayer 的构造函数要求输入样本数，以便一次性申请所有状态向量和矩阵的内存，而不是每输入一个样本就在各状态向量和矩阵后添加一行；改善了几乎所有向量的存储方式，降低了各矩阵的维数，方便理解和调试；优化了训练部分 delta 和 梯度的计算，使用向量外积和点乘来减少 for 循环和求和；添加了输出函数，方便观看构建的神经网络的各参数状态。

import numpy as np

global_epsilon = 1e-3
global_ita = 0.2
#np.random.seed(107)

class SigmoidActivator(object):                                     # 两个激活函数
    def forward(self, weighted_input):
        return 1.0 / (1.0 + np.exp(-weighted_input))
    
    def backward(self, output):
        return output * (1 - output)

class TanhActivator(object):
    def forward(self, weighted_input):
        return 2.0 / (1.0 + np.exp(-2 * weighted_input)) - 1.0
    
    def backward(self, output):
        return 1 - output * output

class LstmLayer(object):
    def __init__(self, sCol, dCol, nSample, ita = global_ita):      # 构造函数要求传入样本数，以申请足够的内存
        self.sCol = sCol
        self.dCol = dCol
        self.ita = ita
        self.nSample = nSample       
        self.time = 0
        self.gActivator = SigmoidActivator()                        # 中间层激活函数
        self.dActivator = TanhActivator()                           # 输出层激活函数
        
        self.f =  np.zeros((self.nSample + 1, self.dCol))           # 初始化状态向量，第 0 行永远为 0，方便递推计算
        self.i =  np.zeros((self.nSample + 1, self.dCol))
        self.ct = np.zeros((self.nSample + 1, self.dCol))
        self.c =  np.zeros((self.nSample + 1, self.dCol))
        self.o =  np.zeros((self.nSample + 1, self.dCol))        
        self.h =  np.zeros((self.nSample + 1, self.dCol))        
        self.Wfh, self.Wfx, self.bf =  self.weightMatrix()        
        self.Wih, self.Wix, self.bi =  self.weightMatrix()        
        self.Wch, self.Wcx, self.bct = self.weightMatrix()
        self.Woh, self.Wox, self.bo =  self.weightMatrix()                

    def weightMatrix(self):                                         # 初始化权重矩阵
        Wh = np.random.uniform(-1, 1,(self.dCol, self.dCol))
        Wx = np.random.uniform(-1, 1,(self.dCol, self.sCol))
        b = np.zeros(self.dCol)                                    
        return Wh, Wx, b

    def forward(self, x):                                           # 前向计算
        self.time += 1
        tt = self.time
        self.f[tt] =  self.gActivator.forward(np.dot(self.Wfh, self.h[tt - 1]) + np.dot(self.Wfx, x) + self.bf)                        
        self.i[tt] =  self.gActivator.forward(np.dot(self.Wih, self.h[tt - 1]) + np.dot(self.Wix, x) + self.bi)
        self.ct[tt] = self.dActivator.forward(np.dot(self.Wch, self.h[tt - 1]) + np.dot(self.Wcx, x) + self.bct)     # 注意 ct 门不一样  
        self.c[tt] =  self.f[tt] * self.c[tt - 1] + self.i[tt] * self.ct[tt]
        self.o[tt] =  self.gActivator.forward(np.dot(self.Woh, self.h[tt - 1]) + np.dot(self.Wox, x) + self.bo)
        self.h[tt] =  self.o[tt] * self.dActivator.forward(self.c[tt])

    def backward(self, x, deltaNextLayer):                          # 后向计算        
        self.deltaF =  np.zeros((self.time + 1, self.dCol))         # 计算误差项部分，初始化各误差项        
        self.deltaI =  np.zeros((self.time + 1, self.dCol))  
        self.deltaO =  np.zeros((self.time + 1, self.dCol))  
        self.deltaCt = np.zeros((self.time + 1, self.dCol))         
        self.deltaH =  np.zeros((self.time + 1, self.dCol))         # deltaH 表示输出的误差项（用于分解给个状态向量）
        self.deltaH[-1] = deltaNextLayer                            # 上一层传递来的误差项                
        for tt in range(self.time, 0, -1):                          # 同层倒序传递
            f =  self.f[tt]
            i =  self.i[tt]            
            ct = self.ct[tt]
            o =  self.o[tt]                                   
            h =  self.deltaH[tt]
            cPre =         self.c[tt-1]
            tanhC =        self.dActivator.forward(self.c[tt])
            inverseTanhC = self.dActivator.backward(tanhC)                       
            
            self.deltaF[tt] =  h * o * inverseTanhC * cPre * self.gActivator.backward(f)                        # 用本层总 dealta 计算各状态向量的 delta
            self.deltaI[tt] =  h * o * inverseTanhC * ct   * self.gActivator.backward(i)
            self.deltaCt[tt] = h * o * inverseTanhC * i    * self.dActivator.backward(ct)
            self.deltaO[tt] =  h     * tanhC               * self.gActivator.backward(o)            
            self.deltaH[tt-1] = np.dot(self.deltaO[tt], self.Woh) + np.dot(self.deltaI[tt], self.Wih) + 
                                np.dot(self.deltaF[tt], self.Wfh) + np.dot(self.deltaCt[tt], self.Wch)          # 用本层个状态向量 delta 计算上层总 delta，这里假设上一层输出即为本层输入，括号外没有偏导数项
                                        
        self.WfhGrad = np.sum(np.array([ np.outer(self.deltaF[1+i], self.h[i]) for i in range(self.time) ]), 0) # 求各状态向量梯度，使用外积完全向量化，类似于张量缩并
        self.WihGrad = np.sum(np.array([ np.outer(self.deltaI[1+i], self.h[i]) for i in range(self.time) ]), 0)
        self.WohGrad = np.sum(np.array([ np.outer(self.deltaO[1+i], self.h[i]) for i in range(self.time) ]), 0)
        self.WchGrad = np.sum(np.array([ np.outer(self.deltaCt[1+i],self.h[i]) for i in range(self.time) ]), 0)
                    
        self.bfGrad = np.sum(self.deltaF[1:1+self.time], 0)         # 求 b 的梯度，相当于上式没有乘法部分
        self.biGrad = np.sum(self.deltaI[1:1+self.time], 0)
        self.boGrad = np.sum(self.deltaO[1:1+self.time], 0)
        self.bcGrad = np.sum(self.deltaCt[1:1+self.time],0)
                                                       
        self.WfxGrad = np.outer(self.deltaF[-1], x)                 # 计算 Wx 的梯度         
        self.WixGrad = np.outer(self.deltaI[-1], x)
        self.WoxGrad = np.outer(self.deltaO[-1], x)
        self.WcxGrad = np.outer(self.deltaCt[-1],x)
    
    def update(self):                                               # 更新权重
        self.Wfh -= self.ita * self.WhfGrad
        self.Wfx -= self.ita * self.WhxGrad
        self.bf -=  self.ita * self.bfGrad
        self.Wih -= self.ita * self.WhiGrad
        self.Wix -= self.ita * self.WhiGrad
        self.bi -=  self.ita * self.biGrad
        self.Woh -= self.ita * self.WofGrad
        self.Wox -= self.ita * self.WoxGrad
        self.bo -=  self.ita * self.boGrad
        self.Wch -= self.ita * self.WcfGrad
        self.Wcx -= self.ita * self.WcxGrad
        self.bct -= self.ita * self.bcGrad            
    
    def reset(self):                                                # 重置各状态向量
        self.time = 0                               
        self.f = np.zeros((self.nSample + 1,self.dCol))
        self.i = np.zeros((self.nSample + 1,self.dCol))        
        self.ct = np.zeros((self.nSample + 1,self.dCol))
        self.c = np.zeros((self.nSample + 1,self.dCol))
        self.o = np.zeros((self.nSample + 1,self.dCol))
        self.h = np.zeros((self.nSample + 1,self.dCol))
                                                                    
    def printLstmLayer(self):                                       # 输出本层神经网络的所有参数
        print("sCol = %d, dCol = %d, ita = %d, nSample = %d, time = %d"%(self.sCol, self.dCol, self.ita, self.nSample, self.time))        
        print("f=
", self.f, "
i=
", self.i, "
ct=
", self.ct, "
c=
", self.c, "
o=
", self.o, "
h=
", self.h)
        print("Wfh=
", self.Wfh, "
Wfx=
", self.Wfx, "
bf=
", self.bf)
        print("Wih=
", self.Wih, "
Wix=
", self.Wix, "
bi=
", self.bi)
        print("Wch=
", self.Wch, "
Wcx=
", self.Wcx, "
bc=
", self.bct)
        print("Woh=
", self.Woh, "
Wox=
", self.Wox, "
bo=
", self.bo)
        
        print("deltaF=
", self.deltaF, "
deltaI=
", self.deltaI, "
deltaO=
", self.deltaO, "
deltaCt=
", self.deltaCt, "
deltaH=
", self.deltaH)        
        print("WfhGrad=
", self.WfhGrad, "
WfxGrad=
", self.WfxGrad, "
bfGrad=
", self.bfGrad)              
        print("WihGrad=
", self.WihGrad, "
WixGrad=
", self.WixGrad, "
biGrad=
", self.biGrad)
        print("WohGrad=
", self.WohGrad, "
WoxGrad=
", self.WoxGrad, "
boGrad=
", self.boGrad)
        print("WchGrad=
", self.WchGrad, "
WcxGrad=
", self.WcxGrad, "
bcGrad=
", self.bcGrad)        
        
def createTestData():                                               # 创建测试数据
    s = [ np.array([1, 2, 3]), np.array([2, 3, 4]) ]
    d = np.array([1, 2])
    return s, d

def test():
    lstmLayer = LstmLayer(3, 2, 2)                                  # 传入输入维度、输出维度，样本数，可选参数学习效率
    x, d = createTestData()
    lstmLayer.forward(x[0]), lstmLayer.forward(x[1])
    lstmLayer.backward(x[1], d)
    lstmLayer.printLstmLayer()

def gradCheck(epsilon = global_epsilon):
    lstm = LstmLayer(3, 2, 2, epsilon)   
    s, d = createTestData()
    lstm.forward(s[0]), lstm.forward(s[1])
    lstm.backward(s[1], np.ones(lstm.h[-1].shape,dtype=np.float64))    # 计算参考梯度，假设最终误差项为全 1 向量    
    for i in range(lstm.Wfh.shape[0]):
        for j in range(lstm.Wfh.shape[1]):
            lstm.Wfh[i,j] += epsilon
            lstm.reset()
            lstm.forward(s[0]), lstm.forward(s[1])
            err1 = np.sum(lstm.h[-1])
            lstm.Wfh[i,j] -= 2*epsilon
            lstm.reset()
            lstm.forward(s[0]), lstm.forward(s[1])
            err2 = np.sum(lstm.h[-1])
            lstm.Wfh[i,j] += epsilon
            print('weights(%d,%d): expected <-> actural %.4e <-> %.4e' % (i, j, (err1 - err2) / (2 * epsilon), lstm.WfhGrad[i,j]))

if __name__ == "__main__":
    test()
    gradCheck()

● 输出结果

sCol = 3, dCol = 2, ita = 0, nSample = 2, time = 2
f=
 [[0.         0.        ]
 [0.08364582 0.64637889]
 [0.04687823 0.75054254]]
i=
 [[0.         0.        ]
 [0.64994358 0.11231909]
 [0.71495984 0.07056953]]
ct=
 [[ 0.          0.        ]
 [-0.77851627 -0.99959223]
 [-0.75373868 -0.99999457]]
c=
 [[ 0.          0.        ]
 [-0.50599165 -0.11227329]
 [-0.56261288 -0.15483503]]
o=
 [[0.         0.        ]
 [0.05723445 0.06130245]
 [0.0173681  0.01644683]]
h=
 [[ 0.          0.        ]
 [-0.02671797 -0.00685385]
 [-0.00885623 -0.00252639]]
Wfh=
 [[-0.44650757 -0.34150997]
 [ 0.1461234   0.7320657 ]]
Wfx=
 [[ 0.61845573 -0.74104458 -0.51005937]
 [ 0.50410244 -0.08955573  0.09272295]]
bf=
 [0. 0.]
Wih=
 [[ 0.05587383 -0.25802153]
 [-0.73662134 -0.25832213]]
Wix=
 [[ 0.33294318 -0.38308928  0.35067554]
 [ 0.03109526  0.40860802 -0.97185992]]
bi=
 [0. 0.]
Wch=
 [[-0.16803787 -0.149016  ]
 [-0.68550217  0.24428858]]
Wcx=
 [[ 0.29142476  0.62232088 -0.85921977]
 [-0.81363189 -0.65205061 -0.71037887]]
bc=
 [0. 0.]
Woh=
 [[-0.09910883 -0.49439315]
 [-0.90781981  0.44788208]]
Wox=
 [[-0.23362093 -0.45101893 -0.55533428]
 [-0.88301662  0.34405375 -0.84458816]]
bo=
 [0. 0.]
deltaF=
 [[ 0.          0.        ]
 [ 0.          0.        ]
 [-0.00029056 -0.00067513]]
deltaI=
 [[ 0.00000000e+00  0.00000000e+00]
 [-4.89958523e-05 -1.29338011e-05]
 [-1.97417511e-03 -2.10655809e-03]]
deltaO=
 [[ 0.00000000e+00  0.00000000e+00]
 [-1.55659244e-04 -1.37923566e-05]
 [-8.70241385e-03 -4.96967325e-03]]
deltaCt=
 [[0.00000000e+00 0.00000000e+00]
 [7.08195651e-05 1.18852402e-08]
 [3.96844066e-03 2.46176389e-08]]
deltaH=
 [[2.28293924e-05 7.62122414e-05]
 [6.17970530e-03 2.14376899e-03]
 [1.00000000e+00 2.00000000e+00]]
WfhGrad=
 [[7.76324944e-06 1.99147518e-06]
 [1.80382086e-05 4.62726915e-06]]
WfxGrad=
 [[-0.00058113 -0.00087169 -0.00116225]
 [-0.00135027 -0.0020254  -0.00270054]]
bfGrad=
 [-0.00029056 -0.00067513]
WihGrad=
 [[5.27459502e-05 1.35307066e-05]
 [5.62829545e-05 1.44380401e-05]]
WixGrad=
 [[-0.00394835 -0.00592253 -0.0078967 ]
 [-0.00421312 -0.00631967 -0.00842623]]
biGrad=
 [-0.00202317 -0.00211949]
WohGrad=
 [[2.32510827e-04 5.96450681e-05]
 [1.32779578e-04 3.40614115e-05]]
WoxGrad=
 [[-0.01740483 -0.02610724 -0.03480966]
 [-0.00993935 -0.01490902 -0.01987869]]
boGrad=
 [-0.00885807 -0.00498347]
WchGrad=
 [[-1.06028676e-04 -2.71991102e-05]
 [-6.57733325e-10 -1.68725686e-10]]
WcxGrad=
 [[7.93688131e-03 1.19053220e-02 1.58737626e-02]
 [4.92352779e-08 7.38529168e-08 9.84705558e-08]]
bcGrad=
 [4.03926022e-03 3.65028791e-08]
weights(0,0): expected <-> actural 1.4570e-02 <-> 1.4570e-02
weights(0,1): expected <-> actural -2.4253e-02 <-> -2.4253e-02
weights(1,0): expected <-> actural -5.2460e-03 <-> -5.2460e-03
weights(1,1): expected <-> actural 8.7327e-03 <-> 8.7327e-03