diff --git a/README.md b/README.md index aa618b0..8250b5b 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,6 @@ The testing file (**test.py**) contains all three networks (network.py, network2 In test.py there are examples of networks configurations with proper comments. I did that to relate with particular chapters from the book. +This is just a test +Second test diff --git a/network.py b/network.py index ad9f26e..6baef54 100644 --- a/network.py +++ b/network.py @@ -70,9 +70,9 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, for mini_batch in mini_batches: self.update_mini_batch(mini_batch, eta) if test_data: - print("Epoch {} : {} / {}".format(j,self.evaluate(test_data),n_test)); + print(f"Epoch {j} : {self.evaluate(test_data)} / {n_test}") else: - print("Epoch {} complete".format(j)) + print(f"Epoch {j} complete") def update_mini_batch(self, mini_batch, eta): """Update the network's weights and biases by applying @@ -135,15 +135,11 @@ def evaluate(self, test_data): return sum(int(x == y) for (x, y) in test_results) def cost_derivative(self, output_activations, y): - """Return the vector of partial derivatives \partial C_x / - \partial a for the output activations.""" return (output_activations-y) #### Miscellaneous functions def sigmoid(z): - """The sigmoid function.""" return 1.0/(1.0+np.exp(-z)) def sigmoid_prime(z): - """Derivative of the sigmoid function.""" return sigmoid(z)*(1-sigmoid(z)) diff --git a/network2.py b/network2.py index 9ecaa5b..abdbe98 100644 --- a/network2.py +++ b/network2.py @@ -1,22 +1,18 @@ -"""network2.py -~~~~~~~~~~~~~~ - +""" An improved version of network.py, implementing the stochastic gradient descent learning algorithm for a feedforward neural network. Improvements include the addition of the cross-entropy cost function, regularization, and better initialization of network weights. Note that I have focused on making the code simple, easily readable, and easily modifiable. It is not optimized, and omits many desirable -features. - -""" +features.""" #### Libraries # Standard library import json import random import sys - +import matplotlib.pyplot as plt # Third-party libraries import numpy as np @@ -100,25 +96,6 @@ def default_weight_initializer(self): self.weights = [np.random.randn(y, x)/np.sqrt(x) for x, y in zip(self.sizes[:-1], self.sizes[1:])] - def large_weight_initializer(self): - """Initialize the weights using a Gaussian distribution with mean 0 - and standard deviation 1. Initialize the biases using a - Gaussian distribution with mean 0 and standard deviation 1. - - Note that the first layer is assumed to be an input layer, and - by convention we won't set any biases for those neurons, since - biases are only ever used in computing the outputs from later - layers. - - This weight and bias initializer uses the same approach as in - Chapter 1, and is included for purposes of comparison. It - will usually be better to use the default weight initializer - instead. - - """ - self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]] - self.weights = [np.random.randn(y, x) - for x, y in zip(self.sizes[:-1], self.sizes[1:])] def feedforward(self, a): """Return the output of the network if ``a`` is input.""" @@ -126,14 +103,13 @@ def feedforward(self, a): a = sigmoid(np.dot(w, a)+b) return a - def SGD(self, training_data, epochs, mini_batch_size, eta, - lmbda = 0.0, + def SGD(self, training_data, epochs, mini_batch_size, eta, n_early_stopping, lmbda = 0.0, evaluation_data=None, monitor_evaluation_cost=False, monitor_evaluation_accuracy=False, monitor_training_cost=False, monitor_training_accuracy=False, - early_stopping_n = 0): + early_stopping = False): """Train the neural network using mini-batch stochastic gradient descent. The ``training_data`` is a list of tuples ``(x, y)`` representing the training inputs and the desired outputs. The @@ -153,10 +129,7 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, are empty if the corresponding flag is not set. """ - - # early stopping functionality: - best_accuracy=1 - + training_data = list(training_data) n = len(training_data) @@ -164,12 +137,10 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, evaluation_data = list(evaluation_data) n_data = len(evaluation_data) - # early stopping functionality: - best_accuracy=0 - no_accuracy_change=0 - evaluation_cost, evaluation_accuracy = [], [] training_cost, training_accuracy = [], [] + max_accuracy = 0 + for j in range(epochs): random.shuffle(training_data) mini_batches = [ @@ -179,40 +150,47 @@ def SGD(self, training_data, epochs, mini_batch_size, eta, self.update_mini_batch( mini_batch, eta, lmbda, len(training_data)) - print("Epoch %s training complete" % j) + print(f"Epoch {j} training complete") + + + #### MONITORING #### if monitor_training_cost: cost = self.total_cost(training_data, lmbda) training_cost.append(cost) - print("Cost on training data: {}".format(cost)) + print(f"Cost on training data: {cost}") if monitor_training_accuracy: accuracy = self.accuracy(training_data, convert=True) training_accuracy.append(accuracy) - print("Accuracy on training data: {} / {}".format(accuracy, n)) + print(f"Accuracy on training data: {accuracy} / {n}") if monitor_evaluation_cost: cost = self.total_cost(evaluation_data, lmbda, convert=True) evaluation_cost.append(cost) - print("Cost on evaluation data: {}".format(cost)) - if monitor_evaluation_accuracy: + print(f"Cost on evaluation data: {cost}") + if monitor_evaluation_accuracy: # Questo parametro viene utilizzato per early stopping accuracy = self.accuracy(evaluation_data) evaluation_accuracy.append(accuracy) - print("Accuracy on evaluation data: {} / {}".format(self.accuracy(evaluation_data), n_data)) - - # Early stopping: - if early_stopping_n > 0: - if accuracy > best_accuracy: - best_accuracy = accuracy - no_accuracy_change = 0 - #print("Early-stopping: Best so far {}".format(best_accuracy)) - else: - no_accuracy_change += 1 - - if (no_accuracy_change == early_stopping_n): - #print("Early-stopping: No accuracy change in last epochs: {}".format(early_stopping_n)) - return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy - - return evaluation_cost, evaluation_accuracy, \ - training_cost, training_accuracy + print(f"Accuracy on evaluation data: {self.accuracy(evaluation_data)} / {n_data}") + + + max_accuracy = max(evaluation_accuracy) + if early_stopping: + print(f'Early stopping is active with no-improvment-in-{n_early_stopping} rule') + print(f'Maximum accuracy in last {n_early_stopping} entries: {max(evaluation_accuracy[-n_early_stopping:])/n_data*100}%') + print(f'Maximum accuracy level reached is {max_accuracy/n_data*100}% \n') + + ### Early Stopping #### + if early_stopping and max(evaluation_accuracy[-n_early_stopping:]) < max_accuracy: + break + + ### PLOT RESULTS #### + plt.plot(evaluation_accuracy) + plt.xlabel('Epochs') + plt.ylabel('Accuracy') + plt.show() + + return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy + def update_mini_batch(self, mini_batch, eta, lmbda, n): """Update the network's weights and biases by applying gradient diff --git a/network3.py b/network3.py index ff8afa8..8033bbe 100644 --- a/network3.py +++ b/network3.py @@ -208,20 +208,25 @@ def __init__(self, filter_shape, image_shape, poolsize=(2, 2), self.image_shape = image_shape self.poolsize = poolsize self.activation_fn=activation_fn + # initialize weights and biases n_out = (filter_shape[0]*np.prod(filter_shape[2:])/np.prod(poolsize)) + self.w = theano.shared( np.asarray( np.random.normal(loc=0, scale=np.sqrt(1.0/n_out), size=filter_shape), dtype=theano.config.floatX), borrow=True) + self.b = theano.shared( np.asarray( np.random.normal(loc=0, scale=1.0, size=(filter_shape[0],)), dtype=theano.config.floatX), borrow=True) + self.params = [self.w, self.b] + def set_inpt(self, inpt, inpt_dropout, mini_batch_size): self.inpt = inpt.reshape(self.image_shape) conv_out = conv.conv2d( diff --git a/test2.py b/test2.py new file mode 100644 index 0000000..2d52d22 --- /dev/null +++ b/test2.py @@ -0,0 +1,31 @@ + +""" + Testing code for different neural network configurations. + Adapted for Python 3.5.2 + + Usage in shell: + python3.5 test.py + + Network (network.py and network2.py) parameters: + 2nd param is epochs count + 3rd param is batch size + 4th param is learning rate (eta) + + Author: + Michał Dobrzański, 2016 + dobrzanski.michal.daniel@gmail.com +""" + +# ---------------------- +# - read the input data: + +import mnist_loader +training_data, validation_data, test_data = mnist_loader.load_data_wrapper() +training_data = list(training_data) + +import network2 + +net = network2.Network([784, 30, 30, 10], cost=network2.CrossEntropyCost) + +net.SGD(training_data, 500, 10, 0.1, 25, 5, evaluation_data=validation_data, + monitor_evaluation_accuracy=True, early_stopping=True)