class Layer ( object ): def __init__ ( self , W_init , b_init , activation ): ''' A layer of a neural network, computes s(Wx + b) where s is a nonlinearity and x is the input vector. :parameters: - W_init : np.ndarray, shape=(n_output, n_input) Values to initialize the weight matrix to. - b_init : np.ndarray, shape=(n_output,) Values to initialize the bias vector - activation : theano.tensor.elemwise.Elemwise Activation function for layer output ''' # Retrieve the input and output dimensionality based on W's initialization n_output , n_input = W_init . shape # Make sure b is n_output in size assert b_init . shape == ( n_output ,) # All parameters should be shared variables. # They're used in this class to compute the layer output, # but are updated elsewhere when optimizing the network parameters. # Note that we are explicitly requiring that W_init has the theano.config.floatX dtype self . W = theano . shared ( value = W_init . astype ( theano . config . floatX ), # The name parameter is solely for printing purporses name = 'W' , # Setting borrow=True allows Theano to use user memory for this object. # It can make code slightly faster by avoiding a deep copy on construction. # For more details, see # http://deeplearning.net/software/theano/tutorial/aliasing.html borrow = True ) # We can force our bias vector b to be a column vector using numpy's reshape method. # When b is a column vector, we can pass a matrix-shaped input to the layer # and get a matrix-shaped output, thanks to broadcasting (described below) self . b = theano . shared ( value = b_init . reshape ( n_output , 1 ) . astype ( theano . config . floatX ), name = 'b' , borrow = True , # Theano allows for broadcasting, similar to numpy. # However, you need to explicitly denote which axes can be broadcasted. # By setting broadcastable=(False, True), we are denoting that b # can be broadcast (copied) along its second dimension in order to be # added to another variable. For more information, see # http://deeplearning.net/software/theano/library/tensor/basic.html broadcastable = ( False , True )) self . activation = activation # We'll compute the gradient of the cost of the network with respect to the parameters in this list. self . params = [ self . W , self . b ] def output ( self , x ): ''' Compute this layer's output given an input :parameters: - x : theano.tensor.var.TensorVariable Theano symbolic variable for layer input :returns: - output : theano.tensor.var.TensorVariable Mixed, biased, and activated x ''' # Compute linear mix lin_output = T . dot ( self . W , x ) + self . b # Output is just linear mix if no activation function # Otherwise, apply the activation function return ( lin_output if self . activation is None else self . activation ( lin_output ))