I tried building a classification neural network using python without any deep learning library. the code runs without error but the cost decreases only at the first iteration and nothing much on the rest of the iterations.
The resulting prediction also only predicts the same class for all test samples. Below are the functions that I made.
def forward_prop(x, w, b):
z = {}
a = {}
a[0] = np.array(x).T
last = len(w)
for i in range(1,(last+1)):
z[i] = w[i-1].dot(a[i-1]) + b[i-1]
if i == last:
a[i] = softmax(z[i])
else:
a[i] = sig(z[i])
pred = a[last]
return pred, a
def back_prop(y, p, a, w, b):
d = {}
d_w = {}
d_b = {}
last = len(w)
d[last] = (p-y.T)
for i in range(last-1, -1, -1):
if i == last-1:
d[i] = w[i].T.dot(d[i+1]) # 10 x 1 --> d2
elif i != 0: # No need to get d0
d[i] = w[i].T.dot(d[i+1]) * sig_deriv(a[i])
d_w[i] = d[i+1].dot(a[i].T) # dw2
d_b[i] = np.sum(d[i+1], axis=1, keepdims=True)
return d_w, d_b
def update(w, b, d_w, d_b, a):
for i in range (0,len(w)):
w[i] = w[i] - a * d_w[i]
b[i] = b[i] - a * d_b[i]
return w, b
def sum_error(y, p):
z = np.sum(y.T * np.log(p))
return here
Activation Function
def sig(x):
return 1.0/(1.0 + np.exp(-x))
def sig_deriv(x):
return x*(1.0-x)
def softmax(x):
#z = np.exp(x-x.max())
z = np.exp(x)
s = z.sum()
out = z / s
return out
Here are the training loop
def train(x, y, w, b, epoch, learn_rate, batch):
c = []
t_v = np.argmax(y, axis=1) # True Value ( 1 / 0 )
err_rate = np.zeros(epoch) # Error rate for each epoch
for i in range(0, epoch):
e = 0 # Reset
p = np.zeros(len(x)) # Prediction ( 1 / 0 )
for k in range(0, len(x)): # Loop through training datasets
pred, layer = forward_prop(x[k:k+batch], w, b)
w_grad, b_grad = back_prop(y[k:k+batch], pred, layer, w, b)
w, b = update(w, b, w_grad, b_grad, learn_rate)
e = e + sum_error(y[k:k+batch], pred)
p[k] = np.argmax(pred, axis=0) # Prediction ( 1 / 0 )
cost = -e/len(x)
c.append(cost)
err_rate[i] = cal_acc(t_v, p)
if (i%(epoch/20)) == 0:
print('iteration:',i,' cost:' ,cost)
return c, err_rate, w, b
And this is the cost plotted
Edit(1) Here are the constants I used
features = range(1,35)
batch_size = 1
epochs = 400
learn_rate = 0.0001
hl = 2 # Numbers of hidden layers
nodes = [20, 20] # Node of each hidden layer
output_node = 2 # Output node
Edit(2) Old initial W and B
def init_model(x, hl, node, p, batch):
w = {}
b = {}
if len(node) < hl:
node = np.tile(node,hl)
n = np.hstack((x,node,p)).ravel() # Number of layer, including in and out
for i in range(0,(len(n)-1)):
w[i] = np.random.rand(n[i+1],n[i]) # row = output ; column = input
b[i] = np.random.rand(n[i+1],batch) # Row = nodes ; column = batch = 1
return w, b
New one
def init_model(x, hl, node, p, batch):
w = {}
b = {}
if len(node) < hl:
node = np.tile(node,hl)
n = np.hstack((x,node,p)).ravel() # Number of layer, including in and out
for i in range(0,(len(n)-1)):
w[i] = np.random.rand(n[i+1],n[i]) - 0.5 # row = output ; column = input
b[i] = np.random.rand(n[i+1],batch) - 0.5 # Row = nodes ; column = batch = 1
return w, b
