import theano
from theano import tensor

from blocks.algorithms import Momentum, AdaDelta
from blocks.bricks import Tanh, Softmax, Linear, MLP
from blocks.bricks.recurrent import LSTM
from blocks.initialization import IsotropicGaussian, Constant

from blocks.filter import VariableFilter
from blocks.roles import WEIGHT
from blocks.graph import ComputationGraph, apply_noise

chars_per_seq = 100
seqs_per_epoch = 1

io_dim = 256

hidden_dims = [200, 500]
activation_function = Tanh()

w_noise_std = 0.01

step_rule = AdaDelta()

pt_freq = 1

param_desc = '' # todo

class Model():
    def __init__(self):
        inp = tensor.lvector('bytes')

        in_onehot = tensor.eq(tensor.arange(io_dim, dtype='int16').reshape((1, io_dim)),
                              inp[:, None])

        dims = [io_dim] + hidden_dims
        prev = in_onehot[None, :, :]
        bricks = []
        for i in xrange(1, len(dims)):
            linear = Linear(input_dim=dims[i-1], output_dim=4*dims[i],
                            name="lstm_in_%d"%i)
            lstm = LSTM(dim=dims[i], activation=activation_function,
                        name="lstm_rec_%d"%i)
            prev = lstm.apply(linear.apply(prev))[0]
            bricks = bricks + [linear, lstm]

        top_linear = MLP(dims=[hidden_dims[-1], io_dim],
                         activations=[Softmax()],
                         name="pred_mlp")
        bricks.append(top_linear)

        out = top_linear.apply(prev.reshape((inp.shape[0], hidden_dims[-1])))

        pred = out.argmax(axis=1)

        cost = Softmax().categorical_cross_entropy(inp[:-1], out[1:])
        error_rate = tensor.neq(inp[:-1], pred[1:]).mean()

        # Initialize
        for brick in bricks:
            brick.weights_init = IsotropicGaussian(0.1)
            brick.biases_init = Constant(0.)
            brick.initialize()

        # apply noise
        cg = ComputationGraph([cost, error_rate])
        noise_vars = VariableFilter(roles=[WEIGHT])(cg)
        cg = apply_noise(cg, noise_vars, w_noise_std)
        [cost_reg, error_rate_reg] = cg.outputs

        self.cost = cost
        self.error_rate = error_rate
        self.cost_reg = cost_reg
        self.error_rate_reg = error_rate_reg
        self.pred = pred