Why do my training losses go up?

Question

I am new to Machine Learning and Tensorflow. For one of my courses, I need to train an MLP for the xor gate. But my losses somehow go up each epoch, which confuses me and I must admit that I ran out of ideas what to do. It would be great if one of the more experienced with Tensorflow could have a look at it. https://drive.google.com/file/d/1QHW_fQSAK8MqIz1D0bb8FypupZ3midZT/view?usp=sharing Best wishes.

import numpy as np
# NEXT LINE ONLY FOR COLAB!
%tensorflow_version 2.x
import tensorflow as tf
import matplotlib.pyplot as plt
# COMMENT OUT THIS LINE FOR COLAB!
%matplotlib notebook


x = np.array([[0,0],[0,1],[1,0],[1,1]], dtype=np.float32)
t = np.array([0,1,1,0], dtype=np.float32)
train_dataset = tf.data.Dataset.from_tensor_slices((x,t))
train_dataset = train_dataset.batch(4)

from tensorflow.keras.layers import Layer

### YOUR CODE HERE ###
# Implement the class for a linear layer.
class Linear(Layer):
    """y = w.x + b"""
    def __init__(self, units):
        super(Linear, self).__init__()
        self.units = units
    def build(self, input_shape):
        self.w = self.add_weight(
                        shape=(input_shape[-1], self.units),
                        initializer=tf.random_normal_initializer(0.0,0.5),
                        trainable=True
        )
        self.b = self.add_weight(
                        shape=(self.units,),
                        initializer=tf.random_normal_initializer(0.0,0.05),
                        trainable=True
        )

    def call(self, inputs):
      return tf.matmul(inputs, self.w) + self.b

### YOUR CODE HERE ###
# Implement the class for the MLP.
class MLP(Layer):

    def __init__(self):
        # And also call the super init again.
        super(MLP, self).__init__()
        # Here we only instantiate the layers that our network has.
        self.hidden_layer = Linear(4)
        self.output_layer = Linear(1)


    def call(self, x):
        x = self.hidden_layer(x)
        x = tf.nn.sigmoid(x)
        x = self.output_layer(x)
        return x

tf.keras.backend.clear_session()


### YOUR CODE HERE ###
# Initialize and train the MLP.

mlp = MLP()

### YOUR CODE HERE ###
% matplotlib inline
plt.figure()
plt.plot(epochs,test_losses)
plt.xlabel("Training Steps")
plt.ylabel("Loss")
plt.xlim()
plt.show()


mse = tf.keras.losses.MeanSquaredError()

optimizer = tf.keras.optimizers.SGD(learning_rate=1)

test_losses = []
epochs = []
accuracies = []


# One epoch means running through the whole dataset once. 
# As we do full batch updates this means we only have on training
# step per epoch. Thus we need many epochs.
for epoch in range(500):
    epochs.append(epoch)

    # Training loop.
    for (x,t) in train_dataset:


        # We have to reshape the input. The input has shape (15,)
        # because we have 15 samples. But if we feed it like that
        # the network thinks we feed in one 15-dimensional input. We want 15
        # 1-dimensional inputs, which would be shape (15,1). 
        # In general the shape of an input should be (batch_size, input_dimension).
        x = tf.reshape(x, shape=(-1,2))



        # We want TensorFlow to automatically compute the gradients
        # for our network. This means we have to start a gradient
        # tape to start recording before we feed the data through
        # the network.

        loss = 0
        with tf.GradientTape() as tape:
            output = mlp(x)
            loss = mse(t, output)
            test_losses.append(loss)
            gradients = tape.gradient(loss, mlp.trainable_variables)


        # After recording the gradients we can apply them to the 
        # variables.
        optimizer.apply_gradients(zip(gradients, mlp.trainable_variables))   

Answer 1

First of all, you need a sigmoid after the output linear layer:

x = self.output_layer(x)
x = tf.nn.sigmoid(x)

Secondly, you also need to reshape the target vector:

t = tf.reshape(t, shape=(-1,1))

Within a few runs, you'll see 100 % accuracy.