1 """ 2 Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy) 3 BSD License 4 """ 5 import numpy as np 6 7 # data I/O 8 data = open('input.txt', 'r', encoding='utf-8').read() # should be simple plain text file 9 chars = list(set(data)) 10 data_size, vocab_size = len(data), len(chars) 11 print('data has %d characters, %d unique.' % (data_size, vocab_size)) 12 char_to_ix = { ch:i for i,ch in enumerate(chars) } 13 ix_to_char = { i:ch for i,ch in enumerate(chars) } 14 15 # hyperparameters 16 hidden_size = 100 # size of hidden layer of neurons 17 seq_length = 25 # number of steps to unroll the RNN for 18 learning_rate = 1e-1 19 20 # model parameters 21 Wxh = np.random.randn(hidden_size, vocab_size)*0.01 # input to hidden 22 Whh = np.random.randn(hidden_size, hidden_size)*0.01 # hidden to hidden 23 Why = np.random.randn(vocab_size, hidden_size)*0.01 # hidden to output 24 bh = np.zeros((hidden_size, 1)) # hidden bias 25 by = np.zeros((vocab_size, 1)) # output bias 26 27 def lossFun(inputs, targets, hprev): 28 """ 29 inputs,targets are both list of integers. 30 hprev is Hx1 array of initial hidden state 31 returns the loss, gradients on model parameters, and last hidden state 32 """ 33 xs, hs, ys, ps = {}, {}, {}, {} 34 hs[-1] = np.copy(hprev) 35 loss = 0 36 # forward pass 37 for t in range(len(inputs)): 38 xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation 39 xs[t][inputs[t]] = 1 40 hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh) # hidden state 41 ys[t] = np.dot(Why, hs[t]) + by # unnormalized log probabilities for next chars 42 ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # probabilities for next chars 43 loss += -np.log(ps[t][targets[t],0]) # softmax (cross-entropy loss) 44 # backward pass: compute gradients going backwards 45 dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why) 46 dbh, dby = np.zeros_like(bh), np.zeros_like(by) 47 dhnext = np.zeros_like(hs[0]) 48 for t in reversed(range(len(inputs))): 49 dy = np.copy(ps[t]) 50 dy[targets[t]] -= 1 # backprop into y. see http://cs231n.github.io/neural-networks-case-study/#grad if confused here 51 dWhy += np.dot(dy, hs[t].T) 52 dby += dy 53 dh = np.dot(Why.T, dy) + dhnext # backprop into h 54 dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity 55 dbh += dhraw 56 dWxh += np.dot(dhraw, xs[t].T) 57 dWhh += np.dot(dhraw, hs[t-1].T) 58 dhnext = np.dot(Whh.T, dhraw) 59 for dparam in [dWxh, dWhh, dWhy, dbh, dby]: 60 np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients 61 return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1] 62 63 def sample(h, seed_ix, n): 64 """ 65 sample a sequence of integers from the model 66 h is memory state, seed_ix is seed letter for first time step 67 """ 68 x = np.zeros((vocab_size, 1)) 69 x[seed_ix] = 1 70 ixes = [] 71 for t in range(n): 72 h = np.tanh(np.dot(Wxh, x) + np.dot(Whh, h) + bh) 73 y = np.dot(Why, h) + by 74 p = np.exp(y) / np.sum(np.exp(y)) 75 ix = np.random.choice(list(range(vocab_size)), p=p.ravel()) 76 x = np.zeros((vocab_size, 1)) 77 x[ix] = 1 78 ixes.append(ix) 79 return ixes 80 81 n, p = 0, 0 82 mWxh, mWhh, mWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why) 83 mbh, mby = np.zeros_like(bh), np.zeros_like(by) # memory variables for Adagrad 84 smooth_loss = -np.log(1.0/vocab_size)*seq_length # loss at iteration 0 85 while True: 86 # prepare inputs (we're sweeping from left to right in steps seq_length long) 87 if p+seq_length+1 >= len(data) or n == 0: 88 hprev = np.zeros((hidden_size,1)) # reset RNN memory 89 p = 0 # go from start of data 90 inputs = [char_to_ix[ch] for ch in data[p:p+seq_length]] 91 targets = [char_to_ix[ch] for ch in data[p+1:p+seq_length+1]] 92 93 # sample from the model now and then 94 if n % 100 == 0: 95 sample_ix = sample(hprev, inputs[0], 200) 96 txt = ''.join(ix_to_char[ix] for ix in sample_ix) 97 print('---- %s ----' % (txt, )) 98 99 # forward seq_length characters through the net and fetch gradient 100 loss, dWxh, dWhh, dWhy, dbh, dby, hprev = lossFun(inputs, targets, hprev) 101 smooth_loss = smooth_loss * 0.999 + loss * 0.001 102 if n % 100 == 0: print('iter %d, loss: %f' % (n, smooth_loss)) # print progress 103 104 # perform parameter update with Adagrad 105 for param, dparam, mem in zip([Wxh, Whh, Why, bh, by], 106 [dWxh, dWhh, dWhy, dbh, dby], 107 [mWxh, mWhh, mWhy, mbh, mby]): 108 mem += dparam * dparam 109 param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update 110 111 p += seq_length # move data pointer 112 n += 1 # iteration counter