0
$\begingroup$

I have been getting into machine learning and I decided to create my own NN Backpropagation program without libraries. I was making progress quite nicely for a while, but I got stuck and I can't seem to figure out the problem with my code. Every problem that woulr require non-linear seperstion, does not work. My assumption is that I made a mistake with the derivatives in the hidden layers.

(Sorry for the quality of the code.)

My program implements:

  • Backpropagation algorithm
  • Cost function: Squared Error
  • Activation function: Sigmoid

It makes use of two classes: FeedForwardLayer and FeedForwardNetwork, the latter containing a list of FeedForwardLayer objects.


Relevant parts of the FeedForwardLayer class:

    int InputNeurons;
    int output;
    internal double[] inputs;
    internal double[] outputs;
    internal double[] Gammas; //stores values that are used for derivation of the hidden layers
    Matrix weight;

public FeedForwardLayer(int n, int output)
    {
        this.output = output;
        outputs = new double[output];
        InputNeurons = n;
        weight = new Matrix(n + 1, output);
        weight.RandomWeight();
        for (int k = 0; k < weight.Col; k++)
        {
            weight[weight.Rows - 1, k] = 0;

        }
        inputs = new double[InputNeurons];
    }
    public FeedForwardLayer(int n)
    {
        output = n;
        outputs = new double[n];
        InputNeurons = n;
        IsOutput = true;
        inputs = new double[InputNeurons];
    }
public double[] ComputeOutput(double[] input_)
    {
        inputs = input_;
        Array.Resize(ref inputs, input_.Length + 1);
        inputs[inputs.Length - 1] = 1; //initializing the bias

        for (int n = 0; n < weight.Rows; n++)
        {
            for (int j = 0; j < weight.Col; j++)
            {
                outputs[j] += inputs[n] * weight[n, j];
            }
        }
        for(int n = 0; n < weight.Col; n++)
        {
             outputs[n] = ActivationFunc.Sigmoid(outputs[n]);
        }
        return outputs;

    }

For the FeedForwardNetwork:

internal List<FeedForwardLayer> layers = new List<FeedForwardLayer>();
    double[] output;
    double totalError;
    double[] deltas; 
    double[][] values; //holds the values of the neurons
public double[] ComputeOutput(double[] input)
    {
        values[0] = input;
        Array.Resize(ref values[0], values[0].Length + 1);
        values[0][values[0].Length - 1] = 1;
        if (firstIteration)
        {
            firstIteration = false;
            for (int n = 0; n < layers.Count; n++)
            {
                if (n == 0)
                {
                    layers[0].IsInput1 = true;

                }
                else if (n != 0 && n + 1 < layers.Count)
                {
                    layers[n].IsHidden1 = true;
                }
                else if (n + 1 == layers.Count)
                {
                    layers[n].IsOutput1 = true;
                }
            }
        }


        layers[0].outputs = layers[0].ComputeOutput(input);
        for (int n = 1; n < layers.Count; n++)
        {
            if (layers[n].IsOutput1)
            {
                layers[n].outputs = layers[n - 1].outputs;
                break;
            }
            layers[n].outputs =layers[n].ComputeOutput(layers[n-1].outputs);

        }

        output = layers[layers.Count-1].outputs;
        return layers[layers.Count-1].outputs;
    }
public void Train(double[] input, double[] expected, double lRate, string Activation)
    {
        CalcDeltas(input, expected);

        double[] Deltas = Delta(); //derivative of the squared error function
        double[] gamma;

        for (int n = NumberOfLayers - 1; n >= 0; n--)
        {
            if (n == NumberOfLayers - 1 || layers[n].IsOutput1)      
            {

                gamma = new double[layers[n - 1].Weight().Col];
                //finding the partial derivative of the cost function in terms of the weight
                //chain rule:  ∂Cost/∂weight = ∂Cost/∂NeuronOut * ∂NOut/∂NIn * ∂NIn/∂w
                //= the derivaties of the cost, activation function and the node multiplied together
                for (int k = 0; k < layers[n - 1].Weight().Col; k++)
                {
                    gamma[k] = -Deltas[k] * ActivationFunctions(Activation, layers[n].outputs[k]);
               //ActivationFunc=derivative of the specified activation function
                }
                layers[n].Gammas = gamma;
                for (int r = 0; r < layers[n - 1].Weight().Rows; r++)
                {
                    for (int c = 0; c < layers[n - 1].Weight().Col; c++)
                    {
                        layers[n - 1].Weight()[r, c] -= gamma[c] * layers[n - 1].inputs[r];
                    }
                }


            }
            else if (n != 0 && n != NumberOfLayers - 1)
            {
                gamma = new double[layers[n - 1].Weight().Col];
                for (int k=0;k< layers[n - 1].Weight().Col; k++)
                {
                    for(int j = 0; j < layers[n + 1].Gammas.Length; j++)
                    {
                        gamma[k] += layers[n + 1].Gammas[j] * layers[n].Weight()[k,j];
                    }
                    gamma[k] *= ActivationFunctions(Activation, layers[n-1].outputs[k]);
                }
                layers[n].Gammas = gamma;
                for (int r = 0; r < layers[n - 1].Weight().Rows; r++)
                {
                    for (int c = 0; c < layers[n - 1].Weight().Col; c++)
                    {
                        layers[n - 1].Weight()[r, c] -= gamma[c] * layers[n - 1].inputs[r];

                    }
                }
            }
        }
    }

Did I make a mistake in the differentiation? According to my calculations the partial derivative of the cost function in the first hidden layer is:

(partial derivative cost function):

for how many output neurons there are. (E1 is the error corresponding to the specified output neuron, the b and f subscripts are the values of the neurons stored before and after the activation function respectively.)

Now, the derivative of the cost function and the activation function is stored in the gamma array initially. Then this gamma value will be updated in the hidden layers by the activation function and the weight connecting the nodes. With each new hidden layer, a new weight and activation function will come in, which we will store in the corresponding gamma value once again. This will go on and on, so theoretically I would only have to multiply the gamma values with the corresponding node.

Unfortunately, It simply does not seem to work. Every problem that would require a non-linear curve returns values that are far from ideal. Did I make a mistake with the hidden layers' differentiation?

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.