I have written a simple digit recognition neural network and it does not seem to be learning. It has 2 hidden layers and uses the softmax activation function and whenever it runs it seems to converge on always picking 0. I would just like to check if the code for updating the weight matrices is correct
from cmath import exp
import numpy as np
from tensorflow.keras.datasets import mnist
class Run:
def __init__(self, num_inputs, num_hidden1, num_hidden2, num_outputs):
self.num_inputs = num_inputs
self.num_hidden1 = num_hidden1
self.num_hidden2 = num_hidden2
self.num_outputs = num_outputs
self.learningrate = 0.001
self.get = GetInput()
self.count = 0
self.countTrue = 0
self.count1 = 0
self.sum = 0
self.past = 0
self.inputLayer = Layer(num_inputs, num_hidden1)
self.hiddenLayer1 = Layer(num_hidden1, num_hidden2)
self.hiddenLayer2 = Layer(num_hidden2, num_outputs)
def getinput(self):
input, expected = self.get.get(self.count)
self.count +=1
self.count1 += 1
return input, expected
def runNN(self, input):
self.inputLayer.calc_output_1(input)
self.hiddenLayer1.calc_output_1(self.inputLayer.fin_outputs)
self.hiddenLayer2.calc_output_1(self.hiddenLayer1.fin_outputs)
self.NN_Output = self.hiddenLayer2.fin_outputs
def calculate_cost(self, expected):
error = 0
for i in range(self.num_outputs):
error += (self.NN_Output[i][0] - expected[i][0])**2 / self.num_outputs
list = []
list1 = []
for each in self.NN_Output:
list.append(float(each[0]))
self.sum += list.index(max(list))
for each in expected:
list1.append(float(each[0]))
if list1.index(max(list1)) == list.index(max(list)):
self.countTrue += 1
print(round(self.countTrue/self.count1, 3))
if self.count1 % 1000 == 0:
print(self.sum / 1000)
print('')
self.past = 0
self.sum = 0
self.count1 = 0
self.countTrue = 0
return error
def calc_new_hidden1(self, expected):
delta = self.NN_Output - expected
change = np.multiply(delta, self.hiddenLayer2.fin_outputs)
change_weights = np.matmul(change, np.transpose(self.hiddenLayer2.inputs)) * self.learningrate
change_bias = change * self.learningrate
self.hiddenLayer2.amend(change_weights, change_bias)
def calc_new_hidden2(self, expected):
delta = self.NN_Output - expected
change = np.multiply(np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta), self.hiddenLayer1.fin_outputs)
change_weights = np.matmul(change, np.transpose(self.hiddenLayer1.inputs)) * self.learningrate
change_bias = change * self.learningrate
self.hiddenLayer1.amend(change_weights, change_bias)
def calc_new_input(self, expected):
delta = (self.NN_Output - expected)
change = np.multiply(np.matmul(np.transpose(self.hiddenLayer1.getter()[0]), np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta)), self.inputLayer.fin_outputs)
change_weights = np.matmul(change, np.transpose(self.inputLayer.inputs)) * self.learningrate
change_bias = change * self.learningrate
self.inputLayer.amend(change_weights, change_bias)
class Layer:
def __init__(self, num_inputs, num_outputs):
self.__weights = np.random.uniform(-0.5, 0.5, (num_outputs, num_inputs))
self.__bias = np.matrix([[float(0)] for x in range(num_outputs)])
def calc_output_1(self, inputs):
self.inputs = inputs
self.__output_1 = np.matmul(self.__weights, inputs) + self.__bias
self.softmax()
def softmax(self):
sum = 0
for each in self.__output_1:
sum += np.exp(float(each[0]))
list1 = []
for each in self.__output_1:
list1.append([float(np.exp(each[0])/sum)])
self.fin_outputs = np.matrix(list1)
def amend(self, change_weights, change_bias):
self.__weights -= change_weights
self.__bias -= change_bias
def getter(self):
return self.__weights, self.__bias
class GetInput:
def __init__(self):
(self.X_train, self.Y_train), (X_test, Y_test) = mnist.load_data()
self.X_train = self.X_train.reshape(self.X_train.shape[0], 28, 28, 1)
x_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
def get(self, i):
list = []
newPhoto = self.X_train[i].astype('float32')/255
for each in newPhoto:
for n in each:
list.append([float(n)])
input = np.matrix(list)
list = []
expect = self.Y_train[i]
for each in range(10):
if each == expect:
list.append([1])
else:
list.append([0])
expected = np.matrix(list)
return input, expected
if __name__ == "__main__":
initiate = Run(784, 600, 400, 10)
while True:
input, expected = initiate.getinput()
initiate.runNN(input)
initiate.calculate_cost(expected)
initiate.calc_new_hidden1(expected)
initiate.calc_new_hidden2(expected)
initiate.calc_new_input(expected)`
Here is the code I have created. The maths for updating the weight matrices is in the Run class: calc_new_hidden1(), calc_new_hidden2(), calc_new_inputs() I think the error will probably be in the calc_new_inputs() function