Merge fa55011 into a7d260b

.travis.yml

@@ -2,6 +2,9 @@ language: python
 python:
   - "2.7"
   - "3.4"
+env:
+  - THEANO_FLAGS=floatX=float32
+  - THEANO_FLAGS=floatX=float64
 before_install:
   - sudo apt-get install -qq libatlas3gf-base libatlas-dev liblapack-dev gfortran
   - wget -q http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
@@ -17,6 +20,6 @@ install:
   - pip install -q coveralls
 script:
   - flake8 blocks
-  - THEANO_FLAGS=blas.ldflags='-lblas -lgfortran' coverage run --source=blocks -m nose2.__main__
+  - THEANO_FLAGS=$THEANO_FLAGS,blas.ldflags='-lblas -lgfortran' coverage run --source=blocks -m nose2.__main__
 after_script:
   - coveralls

blocks/bricks.py

@@ -6,6 +6,7 @@
 import functools
 import logging
 from abc import ABCMeta
+from collections import OrderedDict
 
 import numpy as np
 from theano import tensor
@@ -212,11 +213,7 @@ def allocate(self):
         if not self.allocation_config_pushed:
             self.push_allocation_config()
         for child in self.children:
-            try:
-                child.allocate()
-            except:
-                self.allocation_config_pushed = False
-                raise
+            child.allocate()
         self.params = []
         try:
             self._allocate()
@@ -257,11 +254,7 @@ def initialize(self):
         if not self.initialization_config_pushed:
             self.push_initialization_config()
         for child in self.children:
-            try:
-                child.initialize()
-            except:
-                self.initialization_config_pushed = False
-                raise
+            child.initialize()
         try:
             self._initialize()
         except Exception:
@@ -297,6 +290,12 @@ def push_allocation_config(self):
         """
         self._push_allocation_config()
         self.allocation_config_pushed = True
+        for child in self.children:
+            try:
+                child.push_allocation_config()
+            except:
+                self.allocation_config_pushed = False
+                raise
 
     def _push_allocation_config(self):
         """Brick implementation of configuring child before allocation.
@@ -323,6 +322,12 @@ def push_initialization_config(self):
         """
         self._push_initialization_config()
         self.initialization_config_pushed = True
+        for child in self.children:
+            try:
+                child.push_initialization_config()
+            except:
+                self.initialization_config_pushed = False
+                raise
 
     def _push_initialization_config(self):
         """Brick implementation of configuring child before initialization.
@@ -460,6 +465,10 @@ def __call__(self, *inputs, **kwargs):
                              " list."
                              .format(last, self.brick))
 
+        return_dict = kwargs.pop('return_dict', False)
+        return_list = kwargs.pop('return_list', False)
+        assert not return_list or not return_dict
+
         if not self.brick.allocated:
             self.brick.allocate()
         if not self.brick.initialized and not self.brick.lazy:
@@ -488,6 +497,10 @@ def __call__(self, *inputs, **kwargs):
                 outputs[i] = output.copy()
                 outputs[i].tag.owner = self.brick
                 outputs[i].name = self.brick.name + OUTPUT_SUFFIX
+        if return_list:
+            return outputs
+        if return_dict:
+            return OrderedDict(zip(self.outputs, outputs))
         return unpack(outputs)
 
     def __get__(self, instance, owner):
@@ -643,24 +656,29 @@ def application(application_method):
 
 
 class DefaultRNG(Brick):
-    """A mixin class for Bricks which need a RNG to initialize.
+    """A mixin class for Bricks which need random number generators.
 
     Parameters
     ----------
     rng : object
         A ``numpy.RandomState`` instance.
+    theano_rng : object
+        A ``tensor.shared_randomstreams.RandomStreams`` instance.
 
     Attributes
     ----------
     rng : object
-        If the RNG has been set, return it. Otherwise, return a RNG with a
+        If a RNG has been set, return it. Otherwise, return a RNG with a
         default seed which can be set at a module level using
         ``blocks.bricks.DEFAULT_SEED = seed``.
+    theano_rng : object
+        If a RandomStreams was given in the constructor, return it.
+        Otherwise, return one seeded with ``blocks.bricks.DEFAULT_SEED``.
 
     """
-    def __init__(self, rng=None, **kwargs):
-        self.rng = rng
+    def __init__(self, rng=None, theano_rng=None, **kwargs):
         super(DefaultRNG, self).__init__(**kwargs)
+        update_instance(self, locals())
 
     @property
     def rng(self):
@@ -673,6 +691,17 @@ def rng(self):
     def rng(self, rng):
         self._rng = rng
 
+    @property
+    def theano_rng(self):
+        if getattr(self, '_rng', None) is not None:
+            return self._rng
+        else:
+            return tensor.shared_randomstreams.RandomStreams(DEFAULT_SEED)
+
+    @theano_rng.setter
+    def theano_rng(self, theano_rng):
+        self._theano_rng = theano_rng
+
 
 class Linear(DefaultRNG):
     """A linear transformation with optional bias.
@@ -709,8 +738,8 @@ class Linear(DefaultRNG):
     @lazy
     def __init__(self, input_dim, output_dim, weights_init,
                  biases_init=None, use_bias=True, **kwargs):
-        update_instance(self, locals())
         super(Linear, self).__init__(**kwargs)
+        update_instance(self, locals())
 
     def _allocate(self):
         self.params.append(shared_floatx_zeros((self.input_dim,
@@ -826,6 +855,8 @@ class MLP(DefaultRNG):
     def __init__(self, activations, dims, weights_init, biases_init=None,
                  use_bias=True, **kwargs):
         super(MLP, self).__init__(**kwargs)
+        if activations is None:
+            activations = []
         self.linear_transformations = [Linear(name='linear_{}'.format(i))
                                        for i in range(len(activations))]
         self.children = (self.linear_transformations +

blocks/groundhog/examples/markov_chain.py

@@ -11,14 +11,16 @@
 from groundhog.mainLoop import MainLoop
 from groundhog.trainer.SGD import SGD
 
-from blocks.bricks import GatedRecurrent, Tanh
+from blocks.bricks import Tanh
+from blocks.recurrent import GatedRecurrent
 from blocks.select import Selector
 from blocks.graph import ComputationGraph, Cost
 from blocks.sequence_generators import (
     SequenceGenerator, LinearReadout, SoftmaxEmitter, LookupFeedback)
 from blocks.initialization import Orthogonal, IsotropicGaussian, Constant
 from blocks.groundhog import GroundhogIterator, GroundhogState, GroundhogModel
 from blocks.serialization import load_params
+from blocks.utils import update_instance
 
 floatX = theano.config.floatX
 
@@ -39,8 +41,7 @@ class ChainIterator(GroundhogIterator):
     entropy = equilibrium.dot(trans_entropy).sum()
 
     def __init__(self, rng, seq_len, batch_size):
-        self.__dict__.update(**locals())
-        del self.self
+        update_instance(self, locals())
 
         logger.debug("Markov chain entropy: {}".format(self.entropy))
         logger.debug("Expected min error: {}".format(
@@ -86,9 +87,7 @@ def main():
     num_states = ChainIterator.num_states
     feedback_dim = 8
 
-    transition = GatedRecurrent(
-        name="transition", activation=Tanh(), dim=dim,
-        weights_init=Orthogonal())
+    transition = GatedRecurrent(name="transition", activation=Tanh(), dim=dim)
     generator = SequenceGenerator(
         LinearReadout(readout_dim=num_states, source_names=["states"],
                       emitter=SoftmaxEmitter(name="emitter"),
@@ -109,7 +108,12 @@ def main():
         rng = numpy.random.RandomState(1)
         batch_size = 50
 
+        generator.push_initialization_config()
+        transition.weights_init = Orthogonal()
         generator.initialize()
+        logger.debug("transition.weights_init={}".format(
+            transition.weights_init))
+
         cost = Cost(generator.cost(tensor.lmatrix('x')).sum())
 
         gh_model = GroundhogModel(generator, cost)

blocks/groundhog/examples/sine.py

@@ -15,14 +15,16 @@
     pass  # TODO matplotlib as dependency?
 from theano import tensor
 
-from blocks.bricks import Brick, GatedRecurrent, Identity, Tanh, MLP
+from blocks.bricks import Brick, Identity, Tanh, MLP, lazy, application
+from blocks.recurrent import GatedRecurrent
 from blocks.select import Selector
 from blocks.graph import Cost
 from blocks.sequence_generators import (
-    SequenceGenerator, LinearReadout, TrivialEmitter)
+    SequenceGenerator, LinearReadout, TrivialEmitter, Fork)
 from blocks.initialization import Orthogonal, IsotropicGaussian, Constant
 from blocks.groundhog import GroundhogIterator, GroundhogState, GroundhogModel
 from blocks.serialization import load_params
+from blocks.utils import update_instance
 
 floatX = theano.config.floatX
 logger = logging.getLogger()
@@ -36,33 +38,28 @@ class AddParameters(Brick):
     (e.g. it can be a part of Encoder-Decoder translation model.
 
     """
-    @Brick.lazy_method
+    @lazy
     def __init__(self, transition, num_params, params_name,
                  weights_init, biases_init, **kwargs):
         super(AddParameters, self).__init__(**kwargs)
-        self.__dict__.update(**locals())
-        del self.self
-        del self.kwargs
+        update_instance(self, locals())
 
-        signature = self.transition.apply.signature()
-        self.input_names = signature.forkable_input_names
-        self.state_name = signature.state_names[0]
-        assert len(signature.state_names) == 1
+        self.input_names = [name for name in transition.apply.sequences
+                            if name != 'mask']
+        self.state_name = transition.apply.states[0]
+        assert len(transition.apply.states) == 1
 
-        self.adders = [MLP([Identity()], name="add_{}".format(input_name))
-                       for input_name in self.input_names]
+        self.fork = Fork(self.input_names)
         # Could be also several init bricks, one for each of the states
         self.init = MLP([Identity()], name="init")
-        self.children = [self.transition] + self.adders + [self.init]
+        self.children = [self.transition, self.fork, self.init]
 
     def _push_allocation_config(self):
-        signature = self.transition.apply.signature()
-        for adder, input_name in zip(self.adders, self.input_names):
-            adder.dims[0] = self.num_params
-            adder.dims[-1] = signature.dims[input_name]
+        self.fork.input_dim = self.num_params
+        self.fork.fork_dims = {name: self.transition.get_dim(name)
+                               for name in self.input_names}
         self.init.dims[0] = self.num_params
-        self.init.dims[-1] = signature.dims[signature.state_names[0]]
-        assert len(signature.state_names) == 1
+        self.init.dims[-1] = self.transition.get_dim(self.state_name)
 
     def _push_initialization_config(self):
         for child in self.children:
@@ -71,34 +68,41 @@ def _push_initialization_config(self):
             if self.biases_init:
                 child.biases_init = self.biases_init
 
-    @Brick.apply_method
+    @application
     def apply(self, **kwargs):
         inputs = {name: kwargs.pop(name) for name in self.input_names}
         params = kwargs.pop("params")
-        for name, adder in zip(self.input_names, self.adders):
-            inputs[name] = inputs[name] + adder.apply(params)
+        forks = self.fork.apply(params, return_dict=True)
+        for name in self.input_names:
+            inputs[name] = inputs[name] + forks[name]
         kwargs.update(inputs)
+        if kwargs.get('iterate'):
+            kwargs[self.state_name] = self.initial_state(None, params=params)
         return self.transition.apply(**kwargs)
 
-    @apply.signature_method
-    def apply_signature(self, **kwargs):
-        signature = self.transition.apply.signature()
-        signature.context_names.append(self.params_name)
-        signature.state_init_funcs[self.state_name] = self.initialize_state
-        signature.dims[self.params_name] = self.num_params
-        return signature
+    @apply.delegate
+    def apply_delegate(self):
+        return self.transition.apply
 
-    @Brick.apply_method
-    def initialize_state(self, *args, **kwargs):
-        return self.init.apply(kwargs[self.params_name])
+    @apply.property('contexts')
+    def apply_contexts(self):
+        return [self.params_name] + self.transition.apply.contexts
+
+    @application
+    def initial_state(self, batch_size, *args, **kwargs):
+        return self.init.apply(kwargs['params'])
+
+    def get_dim(self, name):
+        if name == 'params':
+            return self.num_params
+        return self.transition.get_dim(name)
 
 
 class SeriesIterator(GroundhogIterator):
     """Training data generator."""
 
     def __init__(self, rng, func, seq_len, batch_size):
-        self.__dict__.update(**locals())
-        del self.self
+        update_instance(self, locals())
         self.num_params = len(inspect.getargspec(self.func).args) - 1
 
     def next(self):
@@ -149,7 +153,7 @@ def main():
     num_params = len(inspect.getargspec(function).args) - 1
 
     class Emitter(TrivialEmitter):
-        @Brick.apply_method
+        @application
         def cost(self, readouts, outputs):
             """Compute MSE."""
             return ((readouts - outputs) ** 2).sum(axis=readouts.ndim - 1)
@@ -178,27 +182,28 @@ def cost(self, readouts, outputs):
 
         cost = Cost(generator.cost(tensor.tensor3('x'),
                                    params=tensor.matrix("params")).sum())
-        cost.apply_noise(cost.inputs, args.input_noise)
+        if args.input_noise:
+            cost.apply_noise(cost.inputs, args.input_noise)
 
         gh_model = GroundhogModel(generator, cost)
         state = GroundhogState(args.prefix, batch_size,
                                learning_rate=0.0001).as_dict()
         data = SeriesIterator(rng, function, 100, batch_size)
         trainer = SGD(gh_model, state, data)
         main_loop = MainLoop(data, None, None, gh_model, trainer, state, None)
+        main_loop.load()
         main_loop.main()
     elif args.mode == "plot":
         load_params(generator,  args.prefix + "model.npz")
 
         params = tensor.matrix("params")
         sample = theano.function([params], generator.generate(
-            params=params, n_steps=args.steps, batch_size=1, iterate=True))
+            params=params, n_steps=args.steps, batch_size=1))
 
         param_values = numpy.array(map(float, args.params.split()),
                                    dtype=floatX)
-        states, outputs, costs = sample(param_values[None, :])
+        states, outputs, _ = sample(param_values[None, :])
         actual = outputs[:, 0, 0]
-
         desired = numpy.array([function(*(list(param_values) + [T]))
                                for T in range(args.steps)])
         print("MSE: {}".format(((actual - desired) ** 2).sum()))