1

I have written a simple digit recognition neural network and it does not seem to be learning. It has 2 hidden layers and uses the softmax activation function and whenever it runs it seems to converge on always picking 0. I would just like to check if the code for updating the weight matrices is correct

from cmath import exp

import numpy as np

from tensorflow.keras.datasets import mnist



class Run:
    def __init__(self, num_inputs, num_hidden1, num_hidden2, num_outputs):
        self.num_inputs = num_inputs
        self.num_hidden1 = num_hidden1
        self.num_hidden2 = num_hidden2
        self.num_outputs = num_outputs
        self.learningrate = 0.001
        self.get = GetInput()
        self.count = 0
        self.countTrue = 0
        self.count1 = 0 
        self.sum = 0
        self.past = 0

        self.inputLayer = Layer(num_inputs, num_hidden1)
        self.hiddenLayer1 = Layer(num_hidden1, num_hidden2)
        self.hiddenLayer2  = Layer(num_hidden2, num_outputs)

    def getinput(self):
        input, expected = self.get.get(self.count)
        self.count +=1
        self.count1 += 1
        return input, expected

    def runNN(self, input):
        self.inputLayer.calc_output_1(input)
        self.hiddenLayer1.calc_output_1(self.inputLayer.fin_outputs)
        self.hiddenLayer2.calc_output_1(self.hiddenLayer1.fin_outputs)
        self.NN_Output = self.hiddenLayer2.fin_outputs

    def calculate_cost(self, expected):
        error = 0
        for i in range(self.num_outputs):
            error += (self.NN_Output[i][0] - expected[i][0])**2 / self.num_outputs
    
        list = []
        list1 = []
        for each in self.NN_Output:
            list.append(float(each[0]))
    
    
        self.sum += list.index(max(list))
    
        for each in expected:
            list1.append(float(each[0]))

        if list1.index(max(list1)) == list.index(max(list)):
            self.countTrue += 1
    
    
        print(round(self.countTrue/self.count1, 3))

        if self.count1 % 1000 == 0:
            print(self.sum / 1000)
            print('')
            self.past = 0
            self.sum = 0
            self.count1 = 0
            self.countTrue = 0
        return error

    def calc_new_hidden1(self, expected):
        delta = self.NN_Output - expected

        change = np.multiply(delta, self.hiddenLayer2.fin_outputs)


        change_weights = np.matmul(change, np.transpose(self.hiddenLayer2.inputs)) * self.learningrate
        change_bias = change * self.learningrate

        self.hiddenLayer2.amend(change_weights, change_bias)

    def calc_new_hidden2(self, expected):
        delta = self.NN_Output - expected
        change = np.multiply(np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta), self.hiddenLayer1.fin_outputs)

        change_weights = np.matmul(change, np.transpose(self.hiddenLayer1.inputs)) * self.learningrate
        change_bias = change * self.learningrate
        self.hiddenLayer1.amend(change_weights, change_bias)


    def calc_new_input(self, expected):
        delta = (self.NN_Output - expected)

        change = np.multiply(np.matmul(np.transpose(self.hiddenLayer1.getter()[0]), np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta)), self.inputLayer.fin_outputs)

        change_weights = np.matmul(change, np.transpose(self.inputLayer.inputs)) * self.learningrate
        change_bias = change * self.learningrate
        self.inputLayer.amend(change_weights, change_bias)


class Layer:
    def __init__(self, num_inputs, num_outputs):
        self.__weights = np.random.uniform(-0.5, 0.5, (num_outputs, num_inputs))
        self.__bias = np.matrix([[float(0)] for x in range(num_outputs)])

    def calc_output_1(self, inputs):
        self.inputs = inputs
        self.__output_1 = np.matmul(self.__weights, inputs) + self.__bias
        self.softmax()

    def softmax(self):
        sum = 0
        for each in self.__output_1:
            sum += np.exp(float(each[0]))
    
        list1 = []
        for each in self.__output_1:
            list1.append([float(np.exp(each[0])/sum)])
    
        self.fin_outputs = np.matrix(list1)

    def amend(self, change_weights, change_bias):
        self.__weights -= change_weights
        self.__bias -= change_bias

    def getter(self):
        return self.__weights, self.__bias

class GetInput:
    def __init__(self):
        (self.X_train, self.Y_train), (X_test, Y_test) = mnist.load_data()
        self.X_train = self.X_train.reshape(self.X_train.shape[0], 28, 28, 1)
        x_test = X_test.reshape(X_test.shape[0], 28, 28, 1)

    def get(self, i):
        list = []
        newPhoto = self.X_train[i].astype('float32')/255
        for each in newPhoto:
            for n in each:
                list.append([float(n)])
        input = np.matrix(list)

        list = []
        expect = self.Y_train[i]
        for each in range(10):
            if each == expect:
                list.append([1])
            else:
                list.append([0])
        expected = np.matrix(list)
        
        return input, expected

if __name__ == "__main__":
    initiate = Run(784, 600, 400, 10)
    while True:
        input, expected = initiate.getinput()
        initiate.runNN(input)
        initiate.calculate_cost(expected)
        initiate.calc_new_hidden1(expected)
        initiate.calc_new_hidden2(expected)
        initiate.calc_new_input(expected)`

Here is the code I have created. The maths for updating the weight matrices is in the Run class: calc_new_hidden1(), calc_new_hidden2(), calc_new_inputs() I think the error will probably be in the calc_new_inputs() function

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy