I have built a very simple feed-forward neural network which given an input $x \in \{0, 1\}$, it is trained to learn $f(x) = x$, the identity function. Below is a model where on iteration $i$, $x_i$ is the training input, $w_i$ is the connection weight, $A$ is the activation function, and $\hat{y}_i$ is the feed-forward output:
Here is the code:
import numpy as np
# for reproducability
np.random.seed(2017)
def sigmoid(x):
return 1./(1. + np.exp(-x))
def sigmoid_prime(x):
return (1. - sigmoid(x)) * sigmoid(x)
class SimpleNetwork:
def __init__(self):
self.weight = np.random.random()
self.learning_rate = 0.01
self.bias = 1
def predict(self, x):
return sigmoid(x * self.weight + self.bias)
def back_prop(self, x, yh, y, verbose=False):
# compute error
error = 0.5 * (yh - y) ** 2
self.log(error, verbose)
# compute dE/dw
d_weight = (yh - y) * x * sigmoid_prime(self.weight * x + self.bias)
self.weight -= self.learning_rate * d_weight
def log(self, error, verbose=True):
if verbose:
print('error: {}'.format(error))
def fit(self, X, Y, epochs=1, verbose=False):
for _ in range(epochs):
for x, y in zip(X, Y):
yh = self.predict(x)
self.back_prop(x, yh, y, verbose)
X = np.random.randint(0, 2, (100,))
Y = X
net = SimpleNetwork()
net.fit(X, Y, epochs=100, verbose=False)
print(net.predict(0), net.predict(1))
I understand that if the bias = 0, then when $x_i = 0$, $A(x_iw_i) = A(0) = 0.5$, thus the network will never learn the correct output for x = 0. However, when I use bias = 1, the network still converges to a non-zero value. Is there a better bias value I should use?
