Deep Learning Tutorial (翻译) 之 Denoising Autoencoder

英文原文请参考http://www.deeplearning.net/tutorial/dA.html

自编码

一个自编码接受x作为输入，映射成隐藏层表示y：

 

其中，s是非线性函数，如sigmod。y通过一个decoder映射成与x有着相同shape的重构的z，通过相似的转换：

 

z可以看作是x的预测，给定编码code y。可选地，W'可以是W的转置，W‘=W^T，目标是优化参数W，b,b'使得平均重构误差最小。

重构误差取决于输入数据的合适的分布假设，可以使用传统的平方差，如果输入是位变量，可以使用交叉熵：

编码y可以看作是数据的主成分。如果隐藏层是线性的并使用平方差来训练网络，那么k个隐藏层单元可以看作数据的k个主成分。如果隐藏层是非线性的，那么它就不同于PCA。

y可看作x的有损压缩，我们希望优化使得对训练数据和其他输入都有好的压缩效果，但是当测试样本和训练样本不符合同一分布，即相差较大时，效果不好。

我们使用Theano去实现一个自编码器，把它定义成一个类，就可用于构造堆积自编码。

 

隐藏层单元数比输入多的自编码器可以避免学习到同等函数，从而在隐藏层表示捕获关于输入有用的信息。一种方法是增加稀疏性，另一种是增加随机性，这种技术在RBM中被使用到，同样在去噪自编码中使用到，下面介绍去噪自编码。（不太懂）

去噪自编码

In order to force the hidden layer to discover more robust features and prevent it from simply learning the identity, we train the autoencoder to reconstruct the input from a corrupted version of it.（感觉翻译成中文意思就变了）

捕获输入间的统计依赖

去噪自编码可以从多个角度进行理解：多样化学习角度，随机操作角度，置底向上理论角度，等等，所有这些在【Vincent08】有介绍。

在【Vincent08】，随机丢失处理随机地将一些输入变为0。去噪自编码试图从非丢失值中预测丢失值。Note how being able to predict any subset of variables from the rest is a sufficient condition for completely capturing the joint distribution between a set of variables (this is how Gibbs sampling works).

将自编码类转换成去噪自编码类，只需要在输入上增加随机丢失步骤。这里我们randomly masking entries of the input by making them zero.

代码如下，完整代码请参考官方教程。

import numpy
import theano
import os
import sys
import timeit
from theano import tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from logistic_sgd import load_data
from utils import tile_raster_images

try:
    import PIL.Image as Image
except ImportError:
    import Image

class dA(object):
    def __init__(self, numpy_rng, theano_rng=None, input = None,
                 n_visible = 784, n_hidden = 500, W=None, bhid=None, bvis=None):
        '''
        :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
        :param theano_rng: Theano random generator;
        '''
        self.n_visible = n_visible
        self.n_hidden = n_hidden
        if not theano_rng:
            theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
        if not W:
            initial_W = numpy.asarray(
                numpy_rng.uniform(
                    low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)),
                    high=4 * numpy.sqrt(6. / (n_hidden + n_visible)),
                    size=(n_visible, n_hidden)
                ),
                dtype=theano.config.floatX
            )
            W = theano.shared(value=initial_W, name='W', borrow = True)
        if not bvis:
            bvis = theano.shared(
                value=numpy.zeros(n_visible,dtype=theano.config.floatX),
                borrow = True
            )
        if not bhid:
            bhid = theano.shared(
                value=numpy.zeros(n_hidden,dtype=theano.config.floatX),
                borrow = True
            )
        self.W = W
        self.W_prime = self.W.T
        self.b = bhid
        self.b_prime = bvis
        self.theano_rng = theano_rng
        if input is None:
            self.x = T.dmatrix(name='input')
        else:
            self.x = input
        self.params = [self.W, self.b, self.b_prime]

    def get_hidden_values(self, input):
        return T.nnet.sigmoid(T.dot(input, self.W) + self.b)
    def get_reconstructed_input(self, hidden):
        return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime)
    def get_corrupted_input(self, input, corruption_level):
       #binomial()函数为产生0，1的分布，这里是设置产生1的概率为p
       #将输入样本每个像素点以corruption_level的概率被清0
        return self.theano_rng.binomial(size=input.shape, n=1,
                                        p=1-corruption_level,
                                        dtype=theano.config.floatX) * input
    def get_cost_updates(self, corruption_level, learning_rate):
        tilde_x = self.get_corrupted_input(self.x, corruption_level)
        y = self.get_hidden_values(tilde_x)
        z = self.get_reconstructed_input(y)
        L = - T.sum(self.x * T.log(z) + (1 - self.x) * T.log(1-z), axis=1)
        cost = T.mean(L)

        gparams = T.grad(cost, self.params)
        updates = [
            (param, param - learning_rate * gparam)
            for param, gparam in zip(self.params, gparams)
        ]
        return (cost, updates)

def test_dA(learning_rate=0.1, training_epochs=15,
            dataset='mnist.pkl.gz',
            batch_size=20, output_folder='dA_plots'):
    datasets = load_data(dataset)
    train_set_x, train_set_y = datasets[0]
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
    # allocate symbolic variables for the data
    index = T.lscalar()    # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    if not os.path.isdir(output_folder):
        os.makedirs(output_folder)
    os.chdir(output_folder)

    rng = numpy.random.RandomState(123)
    theano_rng = RandomStreams(rng.randint(2 ** 30))
    da = dA(
        numpy_rng=rng,
        theano_rng=theano_rng,
        input=x,
        n_visible=28*28,
        n_hidden=500
    )
    cost, updates = da.get_cost_updates(corruption_level=0.,learning_rate=learning_rate)
    train_da = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size]
        }
    )
    start_time = timeit.default_timer()
    for epoch in range(training_epochs):
        c = []
        for batch_index in range(n_train_batches):
            c.append(train_da(batch_index))
        print('Training epoch %d, cost' %epoch, numpy.mean(c))
    end_time = timeit.default_timer()
    training_time = (end_time - start_time)
    print(('The no corruption code for file ' +
           os.path.split(__file__)[1] +
           ' ran for %.2fm' % ((training_time) / 60.)), file=sys.stderr)
    image = Image.fromarray(
        tile_raster_images(X=da.W.get_value(borrow=True).T,
                           img_shape=(28, 28), tile_shape=(10, 10),
                           tile_spacing=(1, 1)))
    image.save('filters_corruption_0.png')

    os.chdir('../')

if __name__ == '__main__':
    test_dA()

参考资料

1.官方教程

2.实用工具plotting samples and filters