Just for practice, i've tried to create a super-basic neural network script (one neuron for each layer, didn't want to deal with matrices for the sake of simplicity, not that I don't know how to). I've implemented a squared error loss function, sigmoid activation and the derivatives. Of course i created a feedforward function, and a backprop function to calculate the deltas.
import random
import math
LEARNING_RATE = 0.1
TRAINING_EPOCHS = 10_000 #just a reason to make it incompatible with python2
def sigmoid(x):
return 1/(1 + math.exp(-x)) #activation
def der_sigmoid(x):
return x * (1 - x) #derivative for already applied activation
def cost(x):
return x * x #squared error
def der_cost(x):
return 2 * x #derivative of squared error
w1 = 2 * random.random() - 1
w2 = 2 * random.random() - 1 #initializing weights
b1 = 0 #biases
b2 = 0
def feedforward(x, w1, w2, b1, b2):
h = sigmoid((x * w1) + b1)
y = sigmoid((h * w2) + b2) #feedforward
return h, y
def backprop(q_set, a_set, w1, w2, b1, b2):
rand_choice = random.randint(0, len(q_set) - 1)
curr_q = q_set[rand_choice] #choosing random pair from dataset
curr_a = a_set[rand_choice]
h, yHat = feedforward(curr_q, w1, w2, b1, b2) #calculating guess
E = yHat - curr_a #Error
loss = cost(E) #Calculating cost, just because
#print(loss)
gradient_y = der_cost(E) #calculating the derivative of the cost for the output layer
gradient_h = gradient_y * w2 #backpropagating the derivative
delta_w2 = gradient_y * der_sigmoid(yHat) * w2 * LEARNING_RATE
delta_w1 = gradient_h * der_sigmoid(h) * w1 * LEARNING_RATE #calculating weight deltas
delta_b2 = gradient_y * der_sigmoid(yHat) * LEARNING_RATE #bias deltas
delta_b1 = gradient_h * der_sigmoid(h) * LEARNING_RATE
return delta_b1, delta_b2, delta_w1, delta_w2
set_q = [1,2,3,4,5,6,7,8,9]
set_a = [0,1,0,1,0,1,0,1,0] #1 for even, 0 for odd
for i in range(len(set_q)):
set_q[i] /= max(set_q) #normalizing the qset between 0 and 1
print(feedforward(5, w1, w2, b1, b2))
print(feedforward(16, w1, w2, b1, b2)) #testing initial predictions
for i in range(TRAINING_EPOCHS):
delta_b1, delta_b2, delta_w1, delta_w2 = backprop(set_q, set_a, w1, w2, b1, b2)
w1 -= delta_w1
w2 -= delta_w2 #applying deltas
b1 -= delta_b1
b2 -= delta_b2
print(feedforward(5, w1, w2, b1, b2)) #testing
print(feedforward(16, w1, w2, b1, b2)) #why is this output almost the same?
and, as you already guessed, my problem is that even after training, this pseudo-network doesn't converge, instead it keeps giving the same result (or almost) for any input. What's my mistake? (of course I already tried with various parameters and datasets, but still nothing) Thank you in advance