0
$\begingroup$

so I got this working code for a multilayer perceptron class, which I'm training on a XOR dataset (plot 1). As activation I'm using the hyperbolic tangent. After 50000 training epochs using SGD, my network kind of gets the idea and outputs something looking like the XOR dataset (plot 2).

Now I tried to switch the activation from tanh to sigmoid. I also changed my x values to be in range (0.0, 1.0) and my y values to be 1 or 0 instead of 1 or -1, which depends on the x values being >= 0.5 or not. This gives the same XOR dataset, but now in the range of (0.0, 1.0). I switched up the code to use sigmoid, but suddenly the training doesn't work anymore. Any idea why this could be?

Working code using tanh:

import numpy as np
import matplotlib.pyplot as plt

def get_data(n):
    X = np.random.uniform(-1, 1, (n, 2))
    Y = np.array([1 if np.sign(x[0]) != np.sign(x[1]) else -1 for x in X])
    return X, Y


class Multilayer_Perceptron():
def __init__(self, neurons_per_layer):

    #init attributes
    self.layer_count = len(neurons_per_layer)
    self.neurons_per_layer = neurons_per_layer

    #init weights and neurons
    self.w = [np.random.uniform(-1.0, 1.0, (neurons_per_layer[layer + 1], neurons_per_layer[layer] + 1)) for layer in range(self.layer_count - 1)]
    self.a = [np.zeros(neurons_per_layer[layer]) for layer in range(self.layer_count)]
    self.h = [np.zeros(neurons_per_layer[layer]) for layer in range(self.layer_count)]
    self.d = [np.zeros(neurons_per_layer[layer]) for layer in range(self.layer_count)]


def feed_forward(self, X):
    self.h[0] = self.add_bias(X)
    for layer in range(1, self.layer_count):
        for neuron in range(self.neurons_per_layer[layer]):
            self.a[layer][neuron] = self.h[layer - 1].dot(self.w[layer - 1][neuron])
        self.h[layer] = self.add_bias(self.non_linearity_tanh(self.a[layer]))
    return self.h[-1][-1]


def fit(self, X, Y, eta, epochs):
    N = len(Y)

    for epoch in range(epochs):

        #choose random sample for SGD
        sample = np.random.randint(0, N)
        yHat = self.feed_forward(X[sample])
        cost = -2 * (Y[sample] - yHat)

        #calculate the error values d
        self.d[-1][0] = self.non_linearity_deriv_tanh(self.a[-1][0]) * cost
        for layer in range(2, self.layer_count):
            for neuron in range(self.neurons_per_layer[-layer]):
                d_sum = 0
                for next_neuron in range(self.neurons_per_layer[-layer + 1]):
                    d_sum += self.d[-layer + 1][next_neuron] * self.w[-layer + 1][next_neuron][neuron]    
                self.d[-layer][neuron] = self.non_linearity_deriv_tanh(self.a[-layer][neuron]) * d_sum

        #update weights
        for layer in range(self.layer_count - 1):
            for neuron in range(self.neurons_per_layer[layer]):
                for next_neuron in range(self.neurons_per_layer[layer + 1]):
                    self.w[layer][next_neuron][neuron] -= eta * self.d[layer + 1][next_neuron] * self.h[layer][neuron]

def non_linearity_tanh(self, X):
    return (np.e ** X - np.e ** (-X)) / (np.e ** X + np.e ** (-X))


def non_linearity_deriv_tanh(self, X):
    return 1 - self.non_linearity_tanh(X) ** 2


def add_bias(self, X):
    return np.concatenate(([1], X))

multibrain = Multilayer_Perceptron([2, 64, 1])
X, Y = get_data(100000)

fig, ax = plt.subplots(figsize=(10, 10))
ax.scatter(X[:,0], X[:,1], c = ['G' if y == 1 else 'R' for y in Y], s = 3)

multibrain.fit(X, Y, 0.005, 50000)

yHat = [np.sign(multibrain.feed_forward(x)) for x in X]

fig, ax = plt.subplots(figsize=(10, 10))
ax.scatter(X[:,0], X[:,1], c = ['G' if y == 1 else 'R' for y in yHat], s = 3)

plot 1 - XOR dataset plot 2 - network prediction using tanh

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.