diff options
| -rw-r--r-- | lstm.py | 49 | ||||
| -rwxr-xr-x | train.py | 71 |
2 files changed, 83 insertions, 37 deletions
@@ -14,31 +14,34 @@ from blocks.graph import ComputationGraph, apply_noise, apply_dropout # An epoch will be composed of 'num_seqs' sequences of len 'seq_len' # divided in chunks of lengh 'seq_div_size' num_seqs = 20 -seq_len = 2000 -seq_div_size = 100 +seq_len = 5000 +seq_div_size = 200 io_dim = 256 -hidden_dims = [512, 512, 512] +hidden_dims = [1024, 1024, 1024] activation_function = Tanh() i2h_all = True # input to all hidden layers or only first layer h2o_all = True # all hiden layers to output or only last layer -w_noise_std = 0.01 +w_noise_std = 0.02 i_dropout = 0.5 -step_rule = 'momentum' +l1_reg = 0 + +step_rule = 'adadelta' learning_rate = 0.1 momentum = 0.9 -param_desc = '%s-%sIH,%sHO-n%s-d%s-%dx%d(%d)-%s' % ( +param_desc = '%s-%sIH,%sHO-n%s-d%s-l1r%s-%dx%d(%d)-%s' % ( repr(hidden_dims), 'all' if i2h_all else 'first', 'all' if h2o_all else 'last', repr(w_noise_std), repr(i_dropout), + repr(l1_reg), num_seqs, seq_len, seq_div_size, step_rule ) @@ -46,8 +49,9 @@ param_desc = '%s-%sIH,%sHO-n%s-d%s-%dx%d(%d)-%s' % ( save_freq = 5 # parameters for sample generation -sample_len = 60 -sample_temperature = 0.3 +sample_len = 1000 +sample_temperature = 0.7 #0.5 +sample_freq = 10 if step_rule == 'rmsprop': step_rule = RMSProp() @@ -68,9 +72,9 @@ class Model(): # Construct hidden states dims = [io_dim] + hidden_dims - states = [in_onehot.dimshuffle(1, 0, 2)] + hidden = [in_onehot.dimshuffle(1, 0, 2)] bricks = [] - updates = [] + states = [] for i in xrange(1, len(dims)): init_state = theano.shared(numpy.zeros((num_seqs, dims[i])).astype(theano.config.floatX), name='st0_%d'%i) @@ -80,32 +84,32 @@ class Model(): linear = Linear(input_dim=dims[i-1], output_dim=4*dims[i], name="lstm_in_%d"%i) bricks.append(linear) - inter = linear.apply(states[-1]) + inter = linear.apply(hidden[-1]) if i2h_all and i > 1: linear2 = Linear(input_dim=dims[0], output_dim=4*dims[i], name="lstm_in0_%d"%i) bricks.append(linear2) - inter = inter + linear2.apply(states[0]) + inter = inter + linear2.apply(hidden[0]) inter.name = 'inter_bis_%d'%i lstm = LSTM(dim=dims[i], activation=activation_function, name="lstm_rec_%d"%i) bricks.append(lstm) - new_states, new_cells = lstm.apply(inter, + new_hidden, new_cells = lstm.apply(inter, states=init_state, cells=init_cell) - updates.append((init_state, new_states[-1, :, :])) - updates.append((init_cell, new_cells[-1, :, :])) + states.append((init_state, new_hidden[-1, :, :])) + states.append((init_cell, new_cells[-1, :, :])) - states.append(new_states) + hidden.append(new_hidden) - states = [s.dimshuffle(1, 0, 2) for s in states] + hidden = [s.dimshuffle(1, 0, 2) for s in hidden] # Construct output from hidden states out = None - layers = zip(dims, states)[1:] + layers = zip(dims, hidden)[1:] if not h2o_all: layers = [layers[-1]] for i, (dim, state) in enumerate(layers): @@ -136,9 +140,14 @@ class Model(): noise_vars = VariableFilter(roles=[WEIGHT])(cg) cg = apply_noise(cg, noise_vars, w_noise_std) if i_dropout > 0: - cg = apply_dropout(cg, states[1:], i_dropout) + cg = apply_dropout(cg, hidden[1:], i_dropout) [cost_reg, error_rate_reg] = cg.outputs + # add l1 regularization + if l1_reg > 0: + l1pen = sum(abs(st).mean() for st in hidden[1:]) + cost_reg = cost_reg + l1_reg * l1pen + self.cost = cost self.error_rate = error_rate self.cost_reg = cost_reg @@ -146,5 +155,5 @@ class Model(): self.out = out self.pred = pred - self.updates = updates + self.states = states @@ -27,6 +27,8 @@ import datastream logging.basicConfig(level='INFO') logger = logging.getLogger(__name__) +sys.setrecursionlimit(1500) + if __name__ == "__main__": if len(sys.argv) != 2: print >> sys.stderr, 'Usage: %s config' % sys.argv[0] @@ -37,34 +39,64 @@ if __name__ == "__main__": class GenText(SimpleExtension): def __init__(self, model, init_text, max_bytes, **kwargs): + super(GenText, self).__init__(**kwargs) + self.init_text = init_text self.max_bytes = max_bytes - out = model.out[:, -1, :] / numpy.float32(config.sample_temperature) prob = tensor.nnet.softmax(out) cg = ComputationGraph([prob]) assert(len(cg.inputs) == 1) assert(cg.inputs[0].name == 'bytes') - self.f = theano.function(inputs=cg.inputs, outputs=[prob]) - super(GenText, self).__init__(**kwargs) + state_vars = [theano.shared(v[0:1, :].zeros_like().eval(), v.name+'-gen') + for v, _ in model.states] + givens = [(v, x) for (v, _), x in zip(model.states, state_vars)] + updates= [(x, upd) for x, (_, upd) in zip(state_vars, model.states)] + + self.f = theano.function(inputs=cg.inputs, outputs=[prob], + givens=givens, updates=updates) + self.reset_states = theano.function(inputs=[], outputs=[], + updates=[(v, v.zeros_like()) for v in state_vars]) def do(self, which_callback, *args): + + print "Sample:" + print "-------" + + self.reset_states() + v = numpy.array([ord(i) for i in self.init_text], - dtype='int16')[None, :].repeat(axis=0, repeats=config.num_seqs) + dtype='int16')[None, :] + prob, = self.f(v) + sys.stdout.write(self.init_text) while v.shape[1] < self.max_bytes: - prob, = self.f(v) prob = prob / 1.00001 - pred = numpy.zeros((prob.shape[0],), dtype='int16') - for i in range(prob.shape[0]): - pred[i] = numpy.random.multinomial(1, prob[i, :]).nonzero()[0][0] - v = numpy.concatenate([v, pred[:, None]], axis=1) + pred = numpy.random.multinomial(1, prob[0, :]).nonzero()[0][0] + + v = numpy.concatenate([v, pred[None, None]], axis=1) + sys.stdout.write(chr(int(pred))) + sys.stdout.flush() + + prob, = self.f(pred[None, None]) + print + print "-------" + print + - for i in range(v.shape[0]): - print "Sample:", ''.join([chr(int(v[i, j])) for j in range(v.shape[1])]) +class ResetStates(SimpleExtension): + def __init__(self, state_vars, **kwargs): + super(ResetStates, self).__init__(**kwargs) + + self.f = theano.function( + inputs=[], outputs=[], + updates=[(v, v.zeros_like()) for v in state_vars]) + + def do(self, which_callback, *args): + self.f() def train_model(m, train_stream, dump_path=None): @@ -76,17 +108,17 @@ def train_model(m, train_stream, dump_path=None): step_rule=config.step_rule, params=cg.parameters) - algorithm.add_updates(m.updates) + algorithm.add_updates(m.states) # Load the parameters from a dumped model if dump_path is not None: try: - logger.info('Loading parameters...') with closing(numpy.load(dump_path)) as source: + logger.info('Loading parameters...') param_values = {'/' + name.replace(BRICK_DELIMITER, '/'): source[name] for name in source.keys() if name != 'pkl' and not 'None' in name} - model.set_param_values(param_values) + model.set_param_values(param_values) except IOError: pass @@ -96,19 +128,24 @@ def train_model(m, train_stream, dump_path=None): algorithm=algorithm, extensions=[ Checkpoint(path=dump_path, - after_epoch=False, every_n_epochs=config.save_freq), + after_epoch=False, + use_cpickle=True, + every_n_epochs=config.save_freq), TrainingDataMonitoring( [m.cost_reg, m.error_rate_reg, m.cost, m.error_rate], prefix='train', every_n_epochs=1), Printing(every_n_epochs=1, after_epoch=False), - Plot(document='tr_'+model_name+'_'+config.param_desc, + Plot(document='text_'+model_name+'_'+config.param_desc, channels=[['train_cost', 'train_cost_reg'], ['train_error_rate', 'train_error_rate_reg']], server_url='http://eos21:4201/', every_n_epochs=1, after_epoch=False), - GenText(m, ' ', config.sample_len, every_n_epochs=1, after_epoch=False) + GenText(m, '\nalex\ttu crois ?\n', config.sample_len, + every_n_epochs=config.sample_freq, + after_epoch=False, before_training=True), + ResetStates([v for v, _ in m.states], after_epoch=True) ] ) main_loop.run() |