From 62366422f10ba016f256c35d7d6dce4ad64f266c Mon Sep 17 00:00:00 2001 From: Leoric Kong Date: Sat, 9 May 2020 15:13:43 +0800 Subject: [PATCH 1/2] a --- mnist_loader.py | 6 ++++++ mytest.py | 8 ++++++++ network.py | 11 ++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 mytest.py diff --git a/mnist_loader.py b/mnist_loader.py index 320c112..3348d16 100644 --- a/mnist_loader.py +++ b/mnist_loader.py @@ -39,6 +39,9 @@ def load_data(): training_data, validation_data, test_data = pickle.load(f, encoding="latin1") f.close() return (training_data, validation_data, test_data) + # training_data: (images, digits) + # images (50000*784): [ [0,0.1,0.9,0.5,...],... ] + # digits (50000): [5,1,...] def load_data_wrapper(): """Return a tuple containing ``(training_data, validation_data, @@ -67,6 +70,9 @@ def load_data_wrapper(): test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]] test_data = zip(test_inputs, te_d[1]) return (training_data, validation_data, test_data) + # training_data: Iterable of (image, digit) + # image (784,1): T[0,0.1,0.9,0.5,...] + # digits (10,1): T[0,1,0,...] def vectorized_result(j): """Return a 10-dimensional unit vector with a 1.0 in the jth diff --git a/mytest.py b/mytest.py new file mode 100644 index 0000000..a428b16 --- /dev/null +++ b/mytest.py @@ -0,0 +1,8 @@ +import mnist_loader +import network + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper() +#print(training_data) + +net = network.Network([784,100,30,10]) +net.SGD(training_data, 5, 10, 3.0, test_data) diff --git a/network.py b/network.py index ad9f26e..1b9a6aa 100644 --- a/network.py +++ b/network.py @@ -70,7 +70,7 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, for mini_batch in mini_batches: self.update_mini_batch(mini_batch, eta) if test_data: - print("Epoch {} : {} / {}".format(j,self.evaluate(test_data),n_test)); + print("Epoch {} : {} / {}".format(j,self.evaluate(test_data),n_test)) else: print("Epoch {} complete".format(j)) @@ -79,14 +79,23 @@ def update_mini_batch(self, mini_batch, eta): gradient descent using backpropagation to a single mini batch. The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta`` is the learning rate.""" + + # delta_nabla_w: ∂C/∂w + # delta_nabla_b: ∂C/∂b + # nabla_w: Sum(∂C/∂w) + # nabla_b: Sum(∂C/∂b) nabla_b = [np.zeros(b.shape) for b in self.biases] nabla_w = [np.zeros(w.shape) for w in self.weights] for x, y in mini_batch: delta_nabla_b, delta_nabla_w = self.backprop(x, y) nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)] nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)] + + # w' = w - (η/m)*nabla_w = w - (η/m)*Sum(∂C/∂w) self.weights = [w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)] + + # b' = b - (η/m)*nabla_b = b - (η/m)*Sum(∂C/∂b) self.biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)] From 322ace648a12a4aa5d6233f3d8df0bc91be22e3e Mon Sep 17 00:00:00 2001 From: Leoric Kong Date: Thu, 25 Jun 2020 14:38:26 +0800 Subject: [PATCH 2/2] a --- mnist_loader.py | 8 ++++++-- mytest.py | 10 ++++++---- network2.py | 4 ++-- network3.py | 5 ++++- test.py | 6 +++--- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/mnist_loader.py b/mnist_loader.py index 3348d16..70c3c04 100644 --- a/mnist_loader.py +++ b/mnist_loader.py @@ -70,9 +70,13 @@ def load_data_wrapper(): test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]] test_data = zip(test_inputs, te_d[1]) return (training_data, validation_data, test_data) - # training_data: Iterable of (image, digit) + # training_data: Iterable of (image, digit vector) # image (784,1): T[0,0.1,0.9,0.5,...] - # digits (10,1): T[0,1,0,...] + # digit vector (10,1): T[0,0,1,0,...] + + # validation_data: Iterable of (image, digit) + # image (784,1): T[0,0.1,0.9,0.5,...] + # digit: 2 def vectorized_result(j): """Return a 10-dimensional unit vector with a 1.0 in the jth diff --git a/mytest.py b/mytest.py index a428b16..55c2320 100644 --- a/mytest.py +++ b/mytest.py @@ -1,8 +1,10 @@ import mnist_loader -import network +import network2 training_data, validation_data, test_data = mnist_loader.load_data_wrapper() -#print(training_data) +training_data = list(training_data) +validation_data = list(validation_data) -net = network.Network([784,100,30,10]) -net.SGD(training_data, 5, 10, 3.0, test_data) +#training_data = training_data[0:1000] +net = network2.Network([784,30,10]) +net.SGD(training_data, 30, 10, 0.1, lmbda=5, evaluation_data = validation_data, monitor_training_accuracy=True, monitor_evaluation_accuracy = True) diff --git a/network2.py b/network2.py index 9ecaa5b..66fb97b 100644 --- a/network2.py +++ b/network2.py @@ -311,8 +311,8 @@ def total_cost(self, data, lmbda, convert=False): for x, y in data: a = self.feedforward(x) if convert: y = vectorized_result(y) - cost += self.cost.fn(a, y)/len(data) - cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights) # '**' - to the power of. + cost += self.cost.fn(a, y)/len(data) # Original cost + cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights) # with L2 regularization return cost def save(self, filename): diff --git a/network3.py b/network3.py index ff8afa8..ab2f682 100644 --- a/network3.py +++ b/network3.py @@ -154,6 +154,7 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, test_x[i*self.mini_batch_size: (i+1)*self.mini_batch_size] }) # Do the actual training + # Leoric: No shuffling in each epoch? best_validation_accuracy = 0.0 for epoch in range(epochs): for minibatch_index in range(num_training_batches): @@ -209,6 +210,7 @@ def __init__(self, filter_shape, image_shape, poolsize=(2, 2), self.poolsize = poolsize self.activation_fn=activation_fn # initialize weights and biases + # n_out? should be n_in? n_out = (filter_shape[0]*np.prod(filter_shape[2:])/np.prod(poolsize)) self.w = theano.shared( np.asarray( @@ -244,7 +246,7 @@ def __init__(self, n_in, n_out, activation_fn=sigmoid, p_dropout=0.0): self.w = theano.shared( np.asarray( np.random.normal( - loc=0.0, scale=np.sqrt(1.0/n_out), size=(n_in, n_out)), + loc=0.0, scale=np.sqrt(1.0/n_in), size=(n_in, n_out)), dtype=theano.config.floatX), name='w', borrow=True) self.b = theano.shared( @@ -274,6 +276,7 @@ def __init__(self, n_in, n_out, p_dropout=0.0): self.n_out = n_out self.p_dropout = p_dropout # Initialize weights and biases + # w and b for Softmax layer is 0, why not normal distribution? self.w = theano.shared( np.zeros((n_in, n_out), dtype=theano.config.floatX), name='w', borrow=True) diff --git a/test.py b/test.py index d1f4996..ce98a46 100644 --- a/test.py +++ b/test.py @@ -194,7 +194,7 @@ def testTheano(): net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) ''' -# chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy +# chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) and drop out => even better accuracy from network3 import ReLU net = Network([ ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), @@ -205,6 +205,6 @@ def testTheano(): filter_shape=(40, 20, 5, 5), poolsize=(2, 2), activation_fn=ReLU), - FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU), - SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) + FullyConnectedLayer(n_in=40*4*4, n_out=1000, p_dropout=0.5, activation_fn=ReLU), + SoftmaxLayer(n_in=1000, n_out=10)], mini_batch_size) net.SGD(training_data, 60, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1)