Problem Training an LSTM network in Lasagne for simple task (determining parity of bit sequence) I have been trying to gain some familiarity with the Lasagne libraries for machine learning, specifically LSTMs so I set up the following toy problem to determine the parity of a sequence of bits using code from another example.  (I realize that the network is completely overkill for the application).
Whenever I actually start training, the network makes no progress (every attempt is a 50-50 split between the two parity classes).  I have tried varying the hyper-parameters but that doesn't seem to have any effect.
In theory this should be a simple task for the network to learn, so there must be something that I am fundamentally misunderstanding.  Any advice would be greatly appreciated.  Code is below.
from __future__ import print_function


import numpy as np
import theano
import theano.tensor as T
import lasagne

#Lasagne Seed for Reproducibility
lasagne.random.set_rng(np.random.RandomState(1))

# Sequence Length
SEQ_LENGTH = 7

# Number of units in the two hidden (LSTM) layers
N_HIDDEN = 40

# Optimization learning rate
LEARNING_RATE = .1

# All gradients above this will be clipped
GRAD_CLIP = 100

# How often should we check the output?
PRINT_FREQ = 500

# Number of epochs to train the net
NUM_EPOCHS = 50

# Batch Size
BATCH_SIZE = 100

data_size = 100
def buildNetwork():
    print("Building network ...")

    # First, we build the network, starting with an input layer
    # Recurrent layers expect input of shape
    # (batch size, SEQ_LENGTH, num_features)    
    l_in = lasagne.layers.InputLayer(shape=(None, None, 1))

    #Two stacked LSTM layers
    l_forward_1 = lasagne.layers.LSTMLayer(
        l_in, N_HIDDEN, grad_clipping=GRAD_CLIP,
        nonlinearity=lasagne.nonlinearities.tanh)

    l_forward_2 = lasagne.layers.LSTMLayer(
        l_forward_1, N_HIDDEN, grad_clipping=GRAD_CLIP,
        nonlinearity=lasagne.nonlinearities.tanh, only_return_final=True)

    #Slice the output from the LSTM layers to only take the final prediction
    l_forward_slice = lasagne.layers.SliceLayer(l_forward_2, -1, 1)

    l_out = lasagne.layers.DenseLayer(l_forward_2, num_units=2, W = lasagne.init.Normal(), nonlinearity=lasagne.nonlinearities.softmax)

    # Theano tensor for the targets
    target_values = T.ivector('target_output')

    # lasagne.layers.get_output produces a variable for the output of the net
    network_output = lasagne.layers.get_output(l_out)

    # The loss function is calculated as the mean of the (categorical) cross-entropy between the prediction and target.
    cost = T.nnet.categorical_crossentropy(network_output,target_values).mean()

    # Retrieve all parameters from the network
    all_params = lasagne.layers.get_all_params(l_out,trainable=True)

    # Compute AdaGrad updates for training
    print("Computing updates ...")
    updates = lasagne.updates.adagrad(cost, all_params, LEARNING_RATE)

    # Theano functions for training and computing cost
    print("Compiling functions ...")
    train = theano.function([l_in.input_var, target_values], cost, updates=updates, allow_input_downcast=True)
    compute_cost = theano.function([l_in.input_var, target_values], cost, allow_input_downcast=True)

    #Generate a probability distribution
    probs = theano.function([l_in.input_var],network_output,allow_input_downcast=True)

    return (train, compute_cost, probs)

def gen_data(batch_size = BATCH_SIZE, return_target=True):
    #Generate a sequence of 0s and 1s.  Target value is the parity of the sequence
    #i.e. x=0001101 -> y=1  and x=00110011 -> 0 
    x = np.zeros((batch_size,SEQ_LENGTH,1))
    y = np.zeros((batch_size,))

    for n in range(batch_size):
        x[n] = np.random.randint(2, size=(1,SEQ_LENGTH,1))
        if(return_target):
            if (x[n].sum()%2==0):
                y[n] = 0
            else:
                y[n] = 1
    return x, np.array(y,dtype='int32')
def try_it_out(probs):
    #Print a test case during training
    x,y = gen_data(1)
    # Pick the class with highest probability
    ix = np.argmax(probs(x).ravel())
    print("Sequence:",x)
    print("Target:", y)
    print("probs(x)", probs(x).ravel())
    print("Predicted:",ix)


def runIterations(train, compute_cost, probs, num_epochs=NUM_EPOCHS):
    print("Training ...")
    try:
        for it in xrange(data_size * num_epochs / BATCH_SIZE):
            try_it_out(probs) #Run a test

            avg_cost = 0;
            for _ in range(PRINT_FREQ):
                x,y = gen_data()                
                avg_cost += train(x, y)
            print("Epoch {} average loss = {}".format(it*1.0*PRINT_FREQ/data_size*BATCH_SIZE, avg_cost / PRINT_FREQ))

    except KeyboardInterrupt:
        pass
in_vars = buildNetwork()
(train, compute_cost, probs) = in_vars
runIterations(train, compute_cost, probs)

 A: I had similar issue. Take a look at this example for some advice:
https://github.com/Lasagne/Lasagne/blob/master/examples/recurrent.py
The solution that worked for me was to change the cost function by flattening the output, e.g change
cost=T.nnet.categorical_crossentropy(network_output,target_values).mean()

to something like this:
# lasagne.layers.get_output produces a variable for the output of the net
network_output = lasagne.layers.get_output(l_out)

# The network output will have shape (n_batch, 1); let's flatten to get a
# 1-dimensional vector of predicted values
predicted_values = network_output.flatten()

# Our cost will be mean-squared error
cost = T.mean((predicted_values - target_values)**2)

A: Use right init for lstm layers. I use the following initialization:
net = lasagne.layers.LSTMLayer(
    net, 32,
    ingate=lasagne.layers.Gate(
        W_in=Orthogonal(gain=1.5),
        W_hid=Orthogonal(gain=1.5),
        W_cell=Normal(0.1)),
    forgetgate=lasagne.layers.Gate(
        W_in=Orthogonal(gain=1.5),
        W_hid=Orthogonal(gain=1.5),
        W_cell=Normal(0.1)),
    cell=lasagne.layers.Gate(
        W_cell=None,
        nonlinearity=lasagne.nonlinearities.tanh,
        W_in=Orthogonal(gain=1.5),
        W_hid=Orthogonal(gain=1.5)),
    outgate=lasagne.layers.Gate(
        W_in=Orthogonal(gain=1.5),
        W_hid=Orthogonal(gain=1.5),
        W_cell=Normal(0.1)),
    backwards=False)

try it:)
