From cdc25fa076db2c946802ae4e61eeefff99ead97e Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Tue, 29 Jun 2021 08:02:33 -0500 Subject: [PATCH 1/7] mnist_loader.py returns lists instead of zips. network.py and network2.py were modified to accept the lists. network3.py worked without changes. This should make it easier to write hand-crafted data sets, such as the XOR training and testing sets I plan to write. --- mnist_loader.py | 6 +++--- network.py | 2 -- network2.py | 2 -- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mnist_loader.py b/mnist_loader.py index 320c112..b643e3d 100644 --- a/mnist_loader.py +++ b/mnist_loader.py @@ -61,11 +61,11 @@ def load_data_wrapper(): tr_d, va_d, te_d = load_data() training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]] training_results = [vectorized_result(y) for y in tr_d[1]] - training_data = zip(training_inputs, training_results) + training_data = list(zip(training_inputs, training_results)) validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]] - validation_data = zip(validation_inputs, va_d[1]) + validation_data = list(zip(validation_inputs, va_d[1])) test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]] - test_data = zip(test_inputs, te_d[1]) + test_data = list(zip(test_inputs, te_d[1])) return (training_data, validation_data, test_data) def vectorized_result(j): diff --git a/network.py b/network.py index 78629cf..511e85b 100644 --- a/network.py +++ b/network.py @@ -55,11 +55,9 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, epoch, and partial progress printed out. This is useful for tracking progress, but slows things down substantially.""" - training_data = list(training_data) n = len(training_data) if test_data: - test_data = list(test_data) n_test = len(test_data) for j in range(epochs): diff --git a/network2.py b/network2.py index 9ecaa5b..9ed196f 100644 --- a/network2.py +++ b/network2.py @@ -157,11 +157,9 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, # early stopping functionality: best_accuracy=1 - training_data = list(training_data) n = len(training_data) if evaluation_data: - evaluation_data = list(evaluation_data) n_data = len(evaluation_data) # early stopping functionality: From 736ae32a59e4d4db7e21f7de4e50ae279d852087 Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Tue, 29 Jun 2021 08:14:58 -0500 Subject: [PATCH 2/7] ignoring python virtual environment directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bee8a64..4d43738 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ __pycache__ +/venv From 9188d0c07ed29ed29f9611a0cf64e9d581c7c3c5 Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Wed, 7 Jul 2021 13:56:15 -0500 Subject: [PATCH 3/7] uncommenting all tests --- test.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/test.py b/test.py index d1f4996..a77a53f 100644 --- a/test.py +++ b/test.py @@ -17,43 +17,38 @@ # ---------------------- # - read the input data: -''' import mnist_loader training_data, validation_data, test_data = mnist_loader.load_data_wrapper() -training_data = list(training_data) -''' # --------------------- # - network.py example: -#import network +import network -''' net = network.Network([784, 30, 10]) net.SGD(training_data, 30, 10, 3.0, test_data=test_data) -''' # ---------------------- # - network2.py example: -#import network2 +import network2 + -''' net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) #net.large_weight_initializer() net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0,evaluation_data=validation_data, monitor_evaluation_accuracy=True) -''' + # chapter 3 - Overfitting example - too many epochs of learning applied on small (1k samples) amount od data. # Overfitting is treating noise as a signal. -''' + net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) net.large_weight_initializer() net.SGD(training_data[:1000], 400, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True, monitor_training_cost=True) -''' + # chapter 3 - Regularization (weight decay) example 1 (only 1000 of training data and 30 hidden neurons) -''' + net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) net.large_weight_initializer() net.SGD(training_data[:1000], 400, 10, 0.5, @@ -63,10 +58,10 @@ monitor_evaluation_accuracy=True, monitor_training_cost=True, monitor_training_accuracy=True) -''' + # chapter 3 - Early stopping implemented -''' + net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) net.SGD(training_data[:1000], 30, 10, 0.5, lmbda=5.0, @@ -74,17 +69,17 @@ monitor_evaluation_accuracy=True, monitor_training_cost=True, early_stopping_n=10) -''' + # chapter 4 - The vanishing gradient problem - deep networks are hard to train with simple SGD algorithm # this network learns much slower than a shallow one. -''' + net = network2.Network([784, 30, 30, 30, 30, 10], cost=network2.CrossEntropyCost) net.SGD(training_data, 30, 10, 0.1, lmbda=5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True) -''' + # ---------------------- @@ -162,15 +157,15 @@ def testTheano(): mini_batch_size = 10 # chapter 6 - shallow architecture using just a single hidden layer, containing 100 hidden neurons. -''' + net = Network([ FullyConnectedLayer(n_in=784, n_out=100), SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) -''' + # chapter 6 - 5x5 local receptive fields, 20 feature maps, max-pooling layer 2x2 -''' + net = Network([ ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), @@ -178,10 +173,10 @@ def testTheano(): FullyConnectedLayer(n_in=20*12*12, n_out=100), SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) -''' + # chapter 6 - inserting a second convolutional-pooling layer to the previous example => better accuracy -''' + net = Network([ ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), @@ -192,7 +187,7 @@ def testTheano(): FullyConnectedLayer(n_in=40*4*4, n_out=100), SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) -''' + # chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy from network3 import ReLU From 54278f0c2635670633ccc35f68fdb09d75ada27e Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Wed, 7 Jul 2021 13:57:35 -0500 Subject: [PATCH 4/7] Modified network.py to only accept an XOR network I realized that I should delete 'test.py' because the handwritten examples will no longer work. --- XOR_training.py | 19 +++++ network.py | 30 ++++--- test.py | 205 ------------------------------------------------ 3 files changed, 33 insertions(+), 221 deletions(-) create mode 100644 XOR_training.py delete mode 100644 test.py diff --git a/XOR_training.py b/XOR_training.py new file mode 100644 index 0000000..a0a5950 --- /dev/null +++ b/XOR_training.py @@ -0,0 +1,19 @@ +import network +import numpy as np + +training_data = [ + (np.array([[0],[0]], dtype=np.float32), np.array([[1], [0]], dtype=np.float64)), + (np.array([[0],[1]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), + (np.array([[1],[0]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), + (np.array([[1],[1]], dtype=np.float32), np.array([[1], [0]], dtype=np.float64)) +] + +test_data = [ + (np.array([[0],[0]], dtype=np.float32), np.int64(0)), + (np.array([[0],[1]], dtype=np.float32), np.int64(1)), + (np.array([[1],[0]], dtype=np.float32), np.int64(1)), + (np.array([[1],[1]], dtype=np.float32), np.int64(0)) +] + +net = network.Network() +net.SGD(training_data, 10000, 4, 0.4, test_data=test_data) diff --git a/network.py b/network.py index 511e85b..a738489 100644 --- a/network.py +++ b/network.py @@ -21,7 +21,7 @@ class Network(object): - def __init__(self, sizes): + def __init__(self): """The list ``sizes`` contains the number of neurons in the respective layers of the network. For example, if the list was [2, 3, 1] then it would be a three-layer network, with the @@ -32,11 +32,10 @@ def __init__(self, sizes): layer is assumed to be an input layer, and by convention we won't set any biases for those neurons, since biases are only ever used in computing the outputs from later layers.""" - self.num_layers = len(sizes) - self.sizes = sizes - self.biases = [np.random.randn(y, 1) for y in sizes[1:]] - self.weights = [np.random.randn(y, x) - for x, y in zip(sizes[:-1], sizes[1:])] + self.num_layers = 3 + self.sizes = [2, 2, 2] + self.biases = [np.array([[ 1.45722505],[-2.42273809]]), np.array([[ 0.23379489],[-0.13891067]])] + self.weights = [np.array([[-2.89789113, 0.05755303], [-0.27850331, -1.14365775]]),np.array([[-2.18112188, -0.20926775],[ 1.17804109, -0.4288509 ]])] def feedforward(self, a): """Return the output of the network if ``a`` is input.""" @@ -44,8 +43,7 @@ def feedforward(self, a): a = sigmoid(np.dot(w, a)+b) return a - def SGD(self, training_data, epochs, mini_batch_size, eta, - test_data=None): + def SGD(self, training_data, epochs, mini_batch_size, eta, test_data): """Train the neural network using mini-batch stochastic gradient descent. The ``training_data`` is a list of tuples ``(x, y)`` representing the training inputs and the desired @@ -57,8 +55,7 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, n = len(training_data) - if test_data: - n_test = len(test_data) + n_test = len(test_data) for j in range(epochs): random.shuffle(training_data) @@ -67,10 +64,12 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, for k in range(0, n, mini_batch_size)] for mini_batch in mini_batches: self.update_mini_batch(mini_batch, eta) - if test_data: - print("Epoch {} : {} / {}".format(j,self.evaluate(test_data),n_test)) - else: - print("Epoch {} complete".format(j)) + correct_count = self.evaluate(test_data) + print("Epoch {} : {} / {}".format(j, correct_count, n_test)) + if correct_count == n_test: + print(self.biases) + print(self.weights) + return def update_mini_batch(self, mini_batch, eta): """Update the network's weights and biases by applying @@ -128,8 +127,7 @@ def evaluate(self, test_data): network outputs the correct result. Note that the neural network's output is assumed to be the index of whichever neuron in the final layer has the highest activation.""" - test_results = [(np.argmax(self.feedforward(x)), y) - for (x, y) in test_data] + test_results = [(np.argmax(self.feedforward(x)), y) for (x, y) in test_data] return sum(int(x == y) for (x, y) in test_results) def cost_derivative(self, output_activations, y): diff --git a/test.py b/test.py deleted file mode 100644 index a77a53f..0000000 --- a/test.py +++ /dev/null @@ -1,205 +0,0 @@ -""" - Testing code for different neural network configurations. - Adapted for Python 3.5.2 - - Usage in shell: - python3.5 test.py - - Network (network.py and network2.py) parameters: - 2nd param is epochs count - 3rd param is batch size - 4th param is learning rate (eta) - - Author: - Michał Dobrzański, 2016 - dobrzanski.michal.daniel@gmail.com -""" - -# ---------------------- -# - read the input data: -import mnist_loader -training_data, validation_data, test_data = mnist_loader.load_data_wrapper() -# --------------------- -# - network.py example: -import network - -net = network.Network([784, 30, 10]) -net.SGD(training_data, 30, 10, 3.0, test_data=test_data) - -# ---------------------- -# - network2.py example: -import network2 - - -net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) -#net.large_weight_initializer() -net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0,evaluation_data=validation_data, - monitor_evaluation_accuracy=True) - - -# chapter 3 - Overfitting example - too many epochs of learning applied on small (1k samples) amount od data. -# Overfitting is treating noise as a signal. - -net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) -net.large_weight_initializer() -net.SGD(training_data[:1000], 400, 10, 0.5, evaluation_data=test_data, - monitor_evaluation_accuracy=True, - monitor_training_cost=True) - - -# chapter 3 - Regularization (weight decay) example 1 (only 1000 of training data and 30 hidden neurons) - -net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) -net.large_weight_initializer() -net.SGD(training_data[:1000], 400, 10, 0.5, - evaluation_data=test_data, - lmbda = 0.1, # this is a regularization parameter - monitor_evaluation_cost=True, - monitor_evaluation_accuracy=True, - monitor_training_cost=True, - monitor_training_accuracy=True) - - -# chapter 3 - Early stopping implemented - -net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) -net.SGD(training_data[:1000], 30, 10, 0.5, - lmbda=5.0, - evaluation_data=validation_data, - monitor_evaluation_accuracy=True, - monitor_training_cost=True, - early_stopping_n=10) - - -# chapter 4 - The vanishing gradient problem - deep networks are hard to train with simple SGD algorithm -# this network learns much slower than a shallow one. - -net = network2.Network([784, 30, 30, 30, 30, 10], cost=network2.CrossEntropyCost) -net.SGD(training_data, 30, 10, 0.1, - lmbda=5.0, - evaluation_data=validation_data, - monitor_evaluation_accuracy=True) - - - -# ---------------------- -# Theano and CUDA -# ---------------------- - -""" - This deep network uses Theano with GPU acceleration support. - I am using Ubuntu 16.04 with CUDA 7.5. - Tutorial: - http://deeplearning.net/software/theano/install_ubuntu.html#install-ubuntu - - The following command will update only Theano: - sudo pip install --upgrade --no-deps theano - - The following command will update Theano and Numpy/Scipy (warning bellow): - sudo pip install --upgrade theano - -""" - -""" - Below, there is a testing function to check whether your computations have been made on CPU or GPU. - If the result is 'Used the cpu' and you want to have it in gpu, do the following: - 1) install theano: - sudo python3.5 -m pip install Theano - 2) download and install the latest cuda: - https://developer.nvidia.com/cuda-downloads - I had some issues with that, so I followed this idea (better option is to download the 1,1GB package as .run file): - http://askubuntu.com/questions/760242/how-can-i-force-16-04-to-add-a-repository-even-if-it-isnt-considered-secure-eno - You may also want to grab the proper NVidia driver, choose it form there: - System Settings > Software & Updates > Additional Drivers. - 3) should work, run it with: - THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python3.5 test.py - http://deeplearning.net/software/theano/tutorial/using_gpu.html - 4) Optionally, you can add cuDNN support from: - https://developer.nvidia.com/cudnn - - -""" -def testTheano(): - from theano import function, config, shared, sandbox - import theano.tensor as T - import numpy - import time - print("Testing Theano library...") - vlen = 10 * 30 * 768 # 10 x #cores x # threads per core - iters = 1000 - - rng = numpy.random.RandomState(22) - x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) - f = function([], T.exp(x)) - print(f.maker.fgraph.toposort()) - t0 = time.time() - for i in range(iters): - r = f() - t1 = time.time() - print("Looping %d times took %f seconds" % (iters, t1 - t0)) - print("Result is %s" % (r,)) - if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]): - print('Used the cpu') - else: - print('Used the gpu') -# Perform check: -#testTheano() - - -# ---------------------- -# - network3.py example: -import network3 -from network3 import Network, ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer # softmax plus log-likelihood cost is more common in modern image classification networks. - -# read data: -training_data, validation_data, test_data = network3.load_data_shared() -# mini-batch size: -mini_batch_size = 10 - -# chapter 6 - shallow architecture using just a single hidden layer, containing 100 hidden neurons. - -net = Network([ - FullyConnectedLayer(n_in=784, n_out=100), - SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) -net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) - - -# chapter 6 - 5x5 local receptive fields, 20 feature maps, max-pooling layer 2x2 - -net = Network([ - ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), - filter_shape=(20, 1, 5, 5), - poolsize=(2, 2)), - FullyConnectedLayer(n_in=20*12*12, n_out=100), - SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) -net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) - - -# chapter 6 - inserting a second convolutional-pooling layer to the previous example => better accuracy - -net = Network([ - ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), - filter_shape=(20, 1, 5, 5), - poolsize=(2, 2)), - ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12), - filter_shape=(40, 20, 5, 5), - poolsize=(2, 2)), - FullyConnectedLayer(n_in=40*4*4, n_out=100), - SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) -net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) - - -# chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy -from network3 import ReLU -net = Network([ - ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), - filter_shape=(20, 1, 5, 5), - poolsize=(2, 2), - activation_fn=ReLU), - ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12), - filter_shape=(40, 20, 5, 5), - poolsize=(2, 2), - activation_fn=ReLU), - FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU), - SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) -net.SGD(training_data, 60, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1) From 88cfb1277a9024e8ac04d5b1d2494c3e9df422b0 Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Wed, 7 Jul 2021 14:09:14 -0500 Subject: [PATCH 5/7] corresponding test in GNU Octave --- octave/network.m | 4 ++++ octave/sigmoid.m | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 octave/network.m create mode 100644 octave/sigmoid.m diff --git a/octave/network.m b/octave/network.m new file mode 100644 index 0000000..6a30b54 --- /dev/null +++ b/octave/network.m @@ -0,0 +1,4 @@ +W1 = [-2.89914377, 0.05226032; -0.2786159 , -1.14366262] +W2 = [-2.17291276, -0.20878332; 1.17111287, -0.42938584] +b1 = [1.45126723; -2.42262374] +b2 = [0.24568619; -0.14807133] diff --git a/octave/sigmoid.m b/octave/sigmoid.m new file mode 100644 index 0000000..8be6aa1 --- /dev/null +++ b/octave/sigmoid.m @@ -0,0 +1,13 @@ +function g = sigmoid(z) +% SIGMOID Compute sigmoid function +% g = SIGMOID(z) computes the sigmoid of z. + + +% Compute the sigmoid of each value of z (z can be a matrix, +% vector or scalar). + +SIGMOID = @(z) 1./(1 + exp(-z)); + +g = SIGMOID(z); + +end From 1c7407fd53c44d52dd869e01ce171e8763d0bfa0 Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Wed, 7 Jul 2021 16:16:08 -0500 Subject: [PATCH 6/7] negative inputs are zero and positive inputs are one --- XOR_training.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/XOR_training.py b/XOR_training.py index a0a5950..9aa0fd2 100644 --- a/XOR_training.py +++ b/XOR_training.py @@ -2,16 +2,16 @@ import numpy as np training_data = [ - (np.array([[0],[0]], dtype=np.float32), np.array([[1], [0]], dtype=np.float64)), - (np.array([[0],[1]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), - (np.array([[1],[0]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), + (np.array([[-1],[-1]], dtype=np.float32), np.array([[1], [0]], dtype=np.float64)), + (np.array([[-1],[1]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), + (np.array([[1],[-1]], dtype=np.float32), np.array([[0], [1]], dtype=np.float64)), (np.array([[1],[1]], dtype=np.float32), np.array([[1], [0]], dtype=np.float64)) ] test_data = [ - (np.array([[0],[0]], dtype=np.float32), np.int64(0)), - (np.array([[0],[1]], dtype=np.float32), np.int64(1)), - (np.array([[1],[0]], dtype=np.float32), np.int64(1)), + (np.array([[-1],[-1]], dtype=np.float32), np.int64(0)), + (np.array([[-1],[1]], dtype=np.float32), np.int64(1)), + (np.array([[1],[-1]], dtype=np.float32), np.int64(1)), (np.array([[1],[1]], dtype=np.float32), np.int64(0)) ] From 2ed81acf68a1b736dfc05e38e874c736d39f8956 Mon Sep 17 00:00:00 2001 From: Nuclear-Catapult Date: Wed, 7 Jul 2021 16:30:05 -0500 Subject: [PATCH 7/7] changed learning rate to 10 --- XOR_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XOR_training.py b/XOR_training.py index 9aa0fd2..59d6e13 100644 --- a/XOR_training.py +++ b/XOR_training.py @@ -16,4 +16,4 @@ ] net = network.Network() -net.SGD(training_data, 10000, 4, 0.4, test_data=test_data) +net.SGD(training_data, 10000, 4, 10, test_data=test_data)