RNN模拟二进制加法

参考trask大神的文章：https://iamtrask.github.io/2015/11/15/anyone-can-code-lstm

使用numpy实现循环神经网络，以模拟二进制加法为例

"""
CreateTime    : 2019/3/5 21:53
Author        : Aaron
Filename      : RNN_binary_addition.py
"""
import copy
import numpy as np

np.random.seed(3)


def sigmoid(x, derivative=False):
    """non-line function，return derivative when derivative is True"""
    if derivative:
        return x * (1 - x)
    return 1 / (1 + np.exp(-x))


def train():
    # training dataset generation
    int2binary = {}
    binary_dim = 8
    largest_number = pow(2, binary_dim)
    binary = np.unpackbits(
        np.array([range(largest_number)], dtype=np.uint8).T, axis=1)
    for i in range(largest_number):
        int2binary[i] = binary[i]

    # input variables
    alpha = 0.1
    input_dim = 2
    hidden_dim = 16
    output_dim = 1

    # initialize neural network weights and update
    synapse_0 = 2 * np.random.random((input_dim, hidden_dim)) - 1
    synapse_1 = 2 * np.random.random((hidden_dim, output_dim)) - 1
    synapse_h = 2 * np.random.random((hidden_dim, hidden_dim)) - 1

    synapse_0_update = np.zeros_like(synapse_0)
    synapse_1_update = np.zeros_like(synapse_1)
    synapse_h_update = np.zeros_like(synapse_h)

    # training logic
    for j in range(10000):

        # generate a simple addition problem (a + b = c)
        a_int = np.random.randint(largest_number / 2)  # int version
        a = int2binary[a_int]  # binary encoding

        b_int = np.random.randint(largest_number / 2)  # int version
        b = int2binary[b_int]  # binary encoding

        # true answer
        c_int = a_int + b_int
        c = int2binary[c_int]

        # where we'll store our best guess (binary encoded)
        d = np.zeros_like(c)

        overall_error = 0

        layer_2_deltas = list()
        layer_1_values = list()
        layer_1_values.append(np.zeros(hidden_dim))

        # moving along the positions in the binary encoding
        for position in range(binary_dim):
            # generate input and output
            X = np.array([[a[binary_dim - position - 1],
                           b[binary_dim - position - 1]]])
            y = np.array([[c[binary_dim - position - 1]]]).T

            # hidden layer (input ~+ prev_hidden)
            layer_1 = sigmoid(np.dot(X, synapse_0) +
                              np.dot(layer_1_values[-1], synapse_h))

            # output layer (new binary representation)
            layer_2 = sigmoid(np.dot(layer_1, synapse_1))

            # did we miss?... if so, by how much?
            layer_2_error = y - layer_2
            layer_2_deltas.append((layer_2_error) * sigmoid(layer_2, True))
            overall_error += np.abs(layer_2_error[0])

            # decode estimate so we can print it out
            d[binary_dim - position - 1] = np.round(layer_2[0][0])

            # store hidden layer so we can use it in the next time_step
            layer_1_values.append(copy.deepcopy(layer_1))

        future_layer_1_delta = np.zeros(hidden_dim)

        for position in range(binary_dim):
            X = np.array([[a[position], b[position]]])
            layer_1 = layer_1_values[-position - 1]
            prev_layer_1 = layer_1_values[-position - 2]

            # error at output layer
            layer_2_delta = layer_2_deltas[-position - 1]
            # error at hidden layer
            layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) +
                             layer_2_delta.dot(synapse_1.T)) * sigmoid(layer_1,
                                                                       True)

            # let's update all our weights so we can try again
            synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
            synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)
            synapse_0_update += X.T.dot(layer_1_delta)

            future_layer_1_delta = layer_1_delta

        synapse_0 += synapse_0_update * alpha
        synapse_1 += synapse_1_update * alpha
        synapse_h += synapse_h_update * alpha

        synapse_0_update *= 0
        synapse_1_update *= 0
        synapse_h_update *= 0

        # print out progress
        if j % 1000 == 0:
            print("Error:" + str(overall_error))
            print("Pred:" + str(d))
            print("True:" + str(c))
            out = 0
            for index, value in enumerate(reversed(d)):
                out += value * pow(2, index)
            print(str(a_int) + " + " + str(b_int) + " = " + str(out))
            print("------------")


if __name__ == '__main__':
    train()

结果如下

Error:[3.67776144]
Pred:[0 1 0 0 0 0 1 0]
True:[0 1 1 0 0 0 0 0]
66 + 30 = 66
------------
Error:[4.01601049]
Pred:[1 1 1 1 1 1 1 1]
True:[0 0 1 1 0 0 1 1]
50 + 1 = 255
------------
Error:[4.16581017]
Pred:[0 0 0 0 0 0 0 0]
True:[1 1 0 1 1 1 1 0]
118 + 104 = 0
------------
Error:[3.97075811]
Pred:[0 0 0 0 0 0 0 0]
True:[0 1 0 1 0 1 0 1]
29 + 56 = 0
------------
Error:[4.64590095]
Pred:[1 1 1 1 0 1 0 0]
True:[1 0 0 0 1 0 1 0]
123 + 15 = 244
------------
Error:[2.15875271]
Pred:[0 1 0 1 0 1 0 0]
True:[0 1 0 1 0 1 0 0]
32 + 52 = 84
------------
Error:[3.11152336]
Pred:[0 1 1 1 1 1 0 1]
True:[0 1 0 0 0 1 0 1]
62 + 7 = 125
------------
Error:[0.81364924]
Pred:[0 1 1 1 0 1 1 0]
True:[0 1 1 1 0 1 1 0]
83 + 35 = 118
------------
Error:[1.00536035]
Pred:[1 0 0 0 1 0 1 0]
True:[1 0 0 0 1 0 1 0]
95 + 43 = 138
------------
Error:[0.52282946]
Pred:[1 1 0 0 0 1 0 0]
True:[1 1 0 0 0 1 0 0]
82 + 114 = 196
------------