From 68c788e97d78e778ee5695d52e3e82e814d6f6fc Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 26 Mar 2018 11:04:10 +0800 Subject: [PATCH 01/10] 1 --- Chapter_6 BP/bp_test.py | 200 ++++++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/Chapter_6 BP/bp_test.py b/Chapter_6 BP/bp_test.py index 8544ce4..99ba9cd 100644 --- a/Chapter_6 BP/bp_test.py +++ b/Chapter_6 BP/bp_test.py @@ -1,100 +1,100 @@ -# coding:UTF-8 -''' -Date:20160831 -@author: zhaozhiyong -''' -import numpy as np -from bp_train import get_predict - -def load_data(file_name): - '''导入数据 - input: file_name(string):文件的存储位置 - output: feature_data(mat):特征 - ''' - f = open(file_name) # 打开文件 - feature_data = [] - for line in f.readlines(): - feature_tmp = [] - lines = line.strip().split("\t") - for i in xrange(len(lines)): - feature_tmp.append(float(lines[i])) - feature_data.append(feature_tmp) - f.close() # 关闭文件 - return np.mat(feature_data) - -def generate_data(): - '''在[-4.5,4.5]之间随机生成20000组点 - ''' - # 1、随机生成数据点 - data = np.mat(np.zeros((20000, 2))) - m = np.shape(data)[0] - x = np.mat(np.random.rand(20000, 2)) - for i in xrange(m): - data[i, 0] = x[i, 0] * 9 - 4.5 - data[i, 1] = x[i, 1] * 9 - 4.5 - # 2、将数据点保存到文件“test_data”中 - f = open("test_data", "w") - m,n = np.shape(data) - for i in xrange(m): - tmp =[] - for j in xrange(n): - tmp.append(str(data[i,j])) - f.write("\t".join(tmp) + "\n") - f.close() - -def load_model(file_w0, file_w1, file_b0, file_b1): - - def get_model(file_name): - f = open(file_name) - model = [] - for line in f.readlines(): - lines = line.strip().split("\t") - model_tmp = [] - for x in lines: - model_tmp.append(float(x.strip())) - model.append(model_tmp) - f.close() - return np.mat(model) - - # 1、导入输入层到隐含层之间的权重 - w0 = get_model(file_w0) - - # 2、导入隐含层到输出层之间的权重 - w1 = get_model(file_w1) - - # 3、导入输入层到隐含层之间的权重 - b0 = get_model(file_b0) - - # 4、导入隐含层到输出层之间的权重 - b1 = get_model(file_b1) - - return w0, w1, b0, b1 - -def save_predict(file_name, pre): - '''保存最终的预测结果 - input: pre(mat):最终的预测结果 - output: - ''' - f = open(file_name, "w") - m = np.shape(pre)[0] - result = [] - for i in xrange(m): - result.append(str(pre[i, 0])) - f.write("\n".join(result)) - f.close() - -if __name__ == "__main__": - generate_data() - # 1、导入测试数据 - print "--------- 1.load data ------------" - dataTest = load_data("test_data") - # 2、导入BP神经网络模型 - print "--------- 2.load model ------------" - w0, w1, b0, b1 = load_model("weight_w0", "weight_w1", "weight_b0", "weight_b1") - # 3、得到最终的预测值 - print "--------- 3.get prediction ------------" - result = get_predict(dataTest, w0, w1, b0, b1) - # 4、保存最终的预测结果 - print "--------- 4.save result ------------" - pre = np.argmax(result, axis=1) - save_predict("result", pre) +# coding:UTF-8 +''' +Date:20160831 +@author: zhaozhiyong +''' +import numpy as np +from bp_train import get_predict + +def load_data(file_name): + '''导入数据 + input: file_name(string):文件的存储位置 + output: feature_data(mat):特征 + ''' + f = open(file_name) # 打开文件 + feature_data = [] + for line in f.readlines(): + feature_tmp = [] + lines = line.strip().split("\t") + for i in xrange(len(lines)): + feature_tmp.append(float(lines[i])) + feature_data.append(feature_tmp) + f.close() # 关闭文件 + return np.mat(feature_data) + +def generate_data(): + '''在[-4.5,4.5]之间随机生成20000组点 + ''' + # 1、随机生成数据点 + data = np.mat(np.zeros((20000, 2))) + m = np.shape(data)[0] + x = np.mat(np.random.rand(20000, 2)) + for i in xrange(m): + data[i, 0] = x[i, 0] * 9 - 4.5 + data[i, 1] = x[i, 1] * 9 - 4.5 + # 2、将数据点保存到文件“test_data”中 + f = open("test_data", "w") + m,n = np.shape(data) + for i in xrange(m): + tmp =[] + for j in xrange(n): + tmp.append(str(data[i,j])) + f.write("\t".join(tmp) + "\n") + f.close() + +def load_model(file_w0, file_w1, file_b0, file_b1): + + def get_model(file_name): + f = open(file_name) + model = [] + for line in f.readlines(): + lines = line.strip().split("\t") + model_tmp = [] + for x in lines: + model_tmp.append(float(x.strip())) + model.append(model_tmp) + f.close() + return np.mat(model) + + # 1、导入输入层到隐含层之间的权重 + w0 = get_model(file_w0) + + # 2、导入隐含层到输出层之间的权重 + w1 = get_model(file_w1) + + # 3、导入输入层到隐含层之间的权重 + b0 = get_model(file_b0) + + # 4、导入隐含层到输出层之间的权重 + b1 = get_model(file_b1) + + return w0, w1, b0, b1 + +def save_predict(file_name, pre): + '''保存最终的预测结果 + input: pre(mat):最终的预测结果 + output: + ''' + f = open(file_name, "w") + m = np.shape(pre)[0] + result = [] + for i in xrange(m): + result.append(str(pre[i, 0])) + f.write("\n".join(result)) + f.close() + +if __name__ == "__main__": + generate_data() + # 1、导入测试数据 + print ("--------- 1.load data ------------") + dataTest = load_data("test_data") + # 2、导入BP神经网络模型 + print ("--------- 2.load model ------------") + w0, w1, b0, b1 = load_model("weight_w0", "weight_w1", "weight_b0", "weight_b1") + # 3、得到最终的预测值 + print "--------- 3.get prediction ------------" + result = get_predict(dataTest, w0, w1, b0, b1) + # 4、保存最终的预测结果 + print ("--------- 4.save result ------------") + pre = np.argmax(result, axis=1) + save_predict("result", pre) From 09e965661890d709f9a8d13431f24c4344c2f8c1 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 26 Mar 2018 11:05:26 +0800 Subject: [PATCH 02/10] 11 --- Chapter_6 BP/bp_train.py | 474 +++++++++++++++++++-------------------- 1 file changed, 237 insertions(+), 237 deletions(-) diff --git a/Chapter_6 BP/bp_train.py b/Chapter_6 BP/bp_train.py index 446a6fe..3143e53 100644 --- a/Chapter_6 BP/bp_train.py +++ b/Chapter_6 BP/bp_train.py @@ -1,237 +1,237 @@ -# coding:UTF-8 -''' -Date:20160831 -@author: zhaozhiyong -''' -import numpy as np -from math import sqrt - -def load_data(file_name): - '''导入数据 - input: file_name(string):文件的存储位置 - output: feature_data(mat):特征 - label_data(mat):标签 - n_class(int):类别的个数 - ''' - # 1、获取特征 - f = open(file_name) # 打开文件 - feature_data = [] - label_tmp = [] - for line in f.readlines(): - feature_tmp = [] - lines = line.strip().split("\t") - for i in xrange(len(lines) - 1): - feature_tmp.append(float(lines[i])) - label_tmp.append(int(lines[-1])) - feature_data.append(feature_tmp) - f.close() # 关闭文件 - - # 2、获取标签 - m = len(label_tmp) - n_class = len(set(label_tmp)) # 得到类别的个数 - - label_data = np.mat(np.zeros((m, n_class))) - for i in xrange(m): - label_data[i, label_tmp[i]] = 1 - - return np.mat(feature_data), label_data, n_class - -def sig(x): - '''Sigmoid函数 - input: x(mat/float):自变量，可以是矩阵或者是任意实数 - output: Sigmoid值(mat/float):Sigmoid函数的值 - ''' - return 1.0 / (1 + np.exp(-x)) - -def partial_sig(x): - '''Sigmoid导函数的值 - input: x(mat/float):自变量，可以是矩阵或者是任意实数 - output: out(mat/float):Sigmoid导函数的值 - ''' - m, n = np.shape(x) - out = np.mat(np.zeros((m, n))) - for i in xrange(m): - for j in xrange(n): - out[i, j] = sig(x[i, j]) * (1 - sig(x[i, j])) - return out - -def hidden_in(feature, w0, b0): - '''计算隐含层的输入 - input: feature(mat):特征 - w0(mat):输入层到隐含层之间的权重 - b0(mat):输入层到隐含层之间的偏置 - output: hidden_in(mat):隐含层的输入 - ''' - m = np.shape(feature)[0] - hidden_in = feature * w0 - for i in xrange(m): - hidden_in[i, ] += b0 - return hidden_in - -def hidden_out(hidden_in): - '''隐含层的输出 - input: hidden_in(mat):隐含层的输入 - output: hidden_output(mat):隐含层的输出 - ''' - hidden_output = sig(hidden_in) - return hidden_output; - -def predict_in(hidden_out, w1, b1): - '''计算输出层的输入 - input: hidden_out(mat):隐含层的输出 - w1(mat):隐含层到输出层之间的权重 - b1(mat):隐含层到输出层之间的偏置 - output: predict_in(mat):输出层的输入 - ''' - m = np.shape(hidden_out)[0] - predict_in = hidden_out * w1 - for i in xrange(m): - predict_in[i, ] += b1 - return predict_in - -def predict_out(predict_in): - '''输出层的输出 - input: predict_in(mat):输出层的输入 - output: result(mat):输出层的输出 - ''' - result = sig(predict_in) - return result - -def bp_train(feature, label, n_hidden, maxCycle, alpha, n_output): - '''计算隐含层的输入 - input: feature(mat):特征 - label(mat):标签 - n_hidden(int):隐含层的节点个数 - maxCycle(int):最大的迭代次数 - alpha(float):学习率 - n_output(int):输出层的节点个数 - output: w0(mat):输入层到隐含层之间的权重 - b0(mat):输入层到隐含层之间的偏置 - w1(mat):隐含层到输出层之间的权重 - b1(mat):隐含层到输出层之间的偏置 - ''' - m, n = np.shape(feature) - # 1、初始化 - w0 = np.mat(np.random.rand(n, n_hidden)) - w0 = w0 * (8.0 * sqrt(6) / sqrt(n + n_hidden)) - \ - np.mat(np.ones((n, n_hidden))) * \ - (4.0 * sqrt(6) / sqrt(n + n_hidden)) - b0 = np.mat(np.random.rand(1, n_hidden)) - b0 = b0 * (8.0 * sqrt(6) / sqrt(n + n_hidden)) - \ - np.mat(np.ones((1, n_hidden))) * \ - (4.0 * sqrt(6) / sqrt(n + n_hidden)) - w1 = np.mat(np.random.rand(n_hidden, n_output)) - w1 = w1 * (8.0 * sqrt(6) / sqrt(n_hidden + n_output)) - \ - np.mat(np.ones((n_hidden, n_output))) * \ - (4.0 * sqrt(6) / sqrt(n_hidden + n_output)) - b1 = np.mat(np.random.rand(1, n_output)) - b1 = b1 * (8.0 * sqrt(6) / sqrt(n_hidden + n_output)) - \ - np.mat(np.ones((1, n_output))) * \ - (4.0 * sqrt(6) / sqrt(n_hidden + n_output)) - - # 2、训练 - i = 0 - while i <= maxCycle: - # 2.1、信号正向传播 - # 2.1.1、计算隐含层的输入 - hidden_input = hidden_in(feature, w0, b0) # mXn_hidden - # 2.1.2、计算隐含层的输出 - hidden_output = hidden_out(hidden_input) - # 2.1.3、计算输出层的输入 - output_in = predict_in(hidden_output, w1, b1) # mXn_output - # 2.1.4、计算输出层的输出 - output_out = predict_out(output_in) - - # 2.2、误差的反向传播 - # 2.2.1、隐含层到输出层之间的残差 - delta_output = -np.multiply((label - output_out), partial_sig(output_in)) - # 2.2.2、输入层到隐含层之间的残差 - delta_hidden = np.multiply((delta_output * w1.T), partial_sig(hidden_input)) - - # 2.3、 修正权重和偏置 - w1 = w1 - alpha * (hidden_output.T * delta_output) - b1 = b1 - alpha * np.sum(delta_output, axis=0) * (1.0 / m) - w0 = w0 - alpha * (feature.T * delta_hidden) - b0 = b0 - alpha * np.sum(delta_hidden, axis=0) * (1.0 / m) - if i % 100 == 0: - print "\t-------- iter: ", i, \ - " ,cost: ", (1.0/2) * get_cost(get_predict(feature, w0, w1, b0, b1) - label) - i += 1 - return w0, w1, b0, b1 - -def get_cost(cost): - '''计算当前损失函数的值 - input: cost(mat):预测值与标签之间的差 - output: cost_sum / m (double):损失函数的值 - ''' - m,n = np.shape(cost) - - cost_sum = 0.0 - for i in xrange(m): - for j in xrange(n): - cost_sum += cost[i,j] * cost[i,j] - return cost_sum / m - -def get_predict(feature, w0, w1, b0, b1): - '''计算最终的预测 - input: feature(mat):特征 - w0(mat):输入层到隐含层之间的权重 - b0(mat):输入层到隐含层之间的偏置 - w1(mat):隐含层到输出层之间的权重 - b1(mat):隐含层到输出层之间的偏置 - output: 预测值 - ''' - return predict_out(predict_in(hidden_out(hidden_in(feature, w0, b0)), w1, b1)) - -def save_model(w0, w1, b0, b1): - '''保存最终的模型 - input: w0(mat):输入层到隐含层之间的权重 - b0(mat):输入层到隐含层之间的偏置 - w1(mat):隐含层到输出层之间的权重 - b1(mat):隐含层到输出层之间的偏置 - output: - ''' - def write_file(file_name, source): - f = open(file_name, "w") - m, n = np.shape(source) - for i in xrange(m): - tmp = [] - for j in xrange(n): - tmp.append(str(source[i, j])) - f.write("\t".join(tmp) + "\n") - f.close() - - write_file("weight_w0", w0) - write_file("weight_w1", w1) - write_file("weight_b0", b0) - write_file("weight_b1", b1) - -def err_rate(label, pre): - '''计算训练样本上的错误率 - input: label(mat):训练样本的标签 - pre(mat):训练样本的预测值 - output: rate[0,0](float):错误率 - ''' - m = np.shape(label)[0] - err = 0.0 - for i in xrange(m): - if label[i, 0] != pre[i, 0]: - err += 1 - rate = err / m - return rate - -if __name__ == "__main__": - # 1、导入数据 - print "--------- 1.load data ------------" - feature, label, n_class = load_data("data.txt") - # 2、训练网络模型 - print "--------- 2.training ------------" - w0, w1, b0, b1 = bp_train(feature, label, 20, 1000, 0.1, n_class) - # 3、保存最终的模型 - print "--------- 3.save model ------------" - save_model(w0, w1, b0, b1) - # 4、得到最终的预测结果 - print "--------- 4.get prediction ------------" - result = get_predict(feature, w0, w1, b0, b1) - print "训练准确性为：", (1 - err_rate(np.argmax(label, axis=1), np.argmax(result, axis=1))) - +# coding:UTF-8 +''' +Date:20160831 +@author: zhaozhiyong +''' +import numpy as np +from math import sqrt + +def load_data(file_name): + '''导入数据 + input: file_name(string):文件的存储位置 + output: feature_data(mat):特征 + label_data(mat):标签 + n_class(int):类别的个数 + ''' + # 1、获取特征 + f = open(file_name) # 打开文件 + feature_data = [] + label_tmp = [] + for line in f.readlines(): + feature_tmp = [] + lines = line.strip().split("\t") + for i in xrange(len(lines) - 1): + feature_tmp.append(float(lines[i])) + label_tmp.append(int(lines[-1])) + feature_data.append(feature_tmp) + f.close() # 关闭文件 + + # 2、获取标签 + m = len(label_tmp) + n_class = len(set(label_tmp)) # 得到类别的个数 + + label_data = np.mat(np.zeros((m, n_class))) + for i in xrange(m): + label_data[i, label_tmp[i]] = 1 + + return np.mat(feature_data), label_data, n_class + +def sig(x): + '''Sigmoid函数 + input: x(mat/float):自变量，可以是矩阵或者是任意实数 + output: Sigmoid值(mat/float):Sigmoid函数的值 + ''' + return 1.0 / (1 + np.exp(-x)) + +def partial_sig(x): + '''Sigmoid导函数的值 + input: x(mat/float):自变量，可以是矩阵或者是任意实数 + output: out(mat/float):Sigmoid导函数的值 + ''' + m, n = np.shape(x) + out = np.mat(np.zeros((m, n))) + for i in xrange(m): + for j in xrange(n): + out[i, j] = sig(x[i, j]) * (1 - sig(x[i, j])) + return out + +def hidden_in(feature, w0, b0): + '''计算隐含层的输入 + input: feature(mat):特征 + w0(mat):输入层到隐含层之间的权重 + b0(mat):输入层到隐含层之间的偏置 + output: hidden_in(mat):隐含层的输入 + ''' + m = np.shape(feature)[0] + hidden_in = feature * w0 + for i in xrange(m): + hidden_in[i, ] += b0 + return hidden_in + +def hidden_out(hidden_in): + '''隐含层的输出 + input: hidden_in(mat):隐含层的输入 + output: hidden_output(mat):隐含层的输出 + ''' + hidden_output = sig(hidden_in) + return hidden_output; + +def predict_in(hidden_out, w1, b1): + '''计算输出层的输入 + input: hidden_out(mat):隐含层的输出 + w1(mat):隐含层到输出层之间的权重 + b1(mat):隐含层到输出层之间的偏置 + output: predict_in(mat):输出层的输入 + ''' + m = np.shape(hidden_out)[0] + predict_in = hidden_out * w1 + for i in xrange(m): + predict_in[i, ] += b1 + return predict_in + +def predict_out(predict_in): + '''输出层的输出 + input: predict_in(mat):输出层的输入 + output: result(mat):输出层的输出 + ''' + result = sig(predict_in) + return result + +def bp_train(feature, label, n_hidden, maxCycle, alpha, n_output): + '''计算隐含层的输入 + input: feature(mat):特征 + label(mat):标签 + n_hidden(int):隐含层的节点个数 + maxCycle(int):最大的迭代次数 + alpha(float):学习率 + n_output(int):输出层的节点个数 + output: w0(mat):输入层到隐含层之间的权重 + b0(mat):输入层到隐含层之间的偏置 + w1(mat):隐含层到输出层之间的权重 + b1(mat):隐含层到输出层之间的偏置 + ''' + m, n = np.shape(feature) + # 1、初始化 + w0 = np.mat(np.random.rand(n, n_hidden)) + w0 = w0 * (8.0 * sqrt(6) / sqrt(n + n_hidden)) - \ + np.mat(np.ones((n, n_hidden))) * \ + (4.0 * sqrt(6) / sqrt(n + n_hidden)) + b0 = np.mat(np.random.rand(1, n_hidden)) + b0 = b0 * (8.0 * sqrt(6) / sqrt(n + n_hidden)) - \ + np.mat(np.ones((1, n_hidden))) * \ + (4.0 * sqrt(6) / sqrt(n + n_hidden)) + w1 = np.mat(np.random.rand(n_hidden, n_output)) + w1 = w1 * (8.0 * sqrt(6) / sqrt(n_hidden + n_output)) - \ + np.mat(np.ones((n_hidden, n_output))) * \ + (4.0 * sqrt(6) / sqrt(n_hidden + n_output)) + b1 = np.mat(np.random.rand(1, n_output)) + b1 = b1 * (8.0 * sqrt(6) / sqrt(n_hidden + n_output)) - \ + np.mat(np.ones((1, n_output))) * \ + (4.0 * sqrt(6) / sqrt(n_hidden + n_output)) + + # 2、训练 + i = 0 + while i <= maxCycle: + # 2.1、信号正向传播 + # 2.1.1、计算隐含层的输入 + hidden_input = hidden_in(feature, w0, b0) # mXn_hidden + # 2.1.2、计算隐含层的输出 + hidden_output = hidden_out(hidden_input) + # 2.1.3、计算输出层的输入 + output_in = predict_in(hidden_output, w1, b1) # mXn_output + # 2.1.4、计算输出层的输出 + output_out = predict_out(output_in) + + # 2.2、误差的反向传播 + # 2.2.1、隐含层到输出层之间的残差 + delta_output = -np.multiply((label - output_out), partial_sig(output_in)) + # 2.2.2、输入层到隐含层之间的残差 + delta_hidden = np.multiply((delta_output * w1.T), partial_sig(hidden_input)) + + # 2.3、 修正权重和偏置 + w1 = w1 - alpha * (hidden_output.T * delta_output) + b1 = b1 - alpha * np.sum(delta_output, axis=0) * (1.0 / m) + w0 = w0 - alpha * (feature.T * delta_hidden) + b0 = b0 - alpha * np.sum(delta_hidden, axis=0) * (1.0 / m) + if i % 100 == 0: + print ("\t-------- iter: ", i, \ + " ,cost: ", (1.0/2) * get_cost(get_predict(feature, w0, w1, b0, b1) - label)) + i += 1 + return w0, w1, b0, b1 + +def get_cost(cost): + '''计算当前损失函数的值 + input: cost(mat):预测值与标签之间的差 + output: cost_sum / m (double):损失函数的值 + ''' + m,n = np.shape(cost) + + cost_sum = 0.0 + for i in xrange(m): + for j in xrange(n): + cost_sum += cost[i,j] * cost[i,j] + return cost_sum / m + +def get_predict(feature, w0, w1, b0, b1): + '''计算最终的预测 + input: feature(mat):特征 + w0(mat):输入层到隐含层之间的权重 + b0(mat):输入层到隐含层之间的偏置 + w1(mat):隐含层到输出层之间的权重 + b1(mat):隐含层到输出层之间的偏置 + output: 预测值 + ''' + return predict_out(predict_in(hidden_out(hidden_in(feature, w0, b0)), w1, b1)) + +def save_model(w0, w1, b0, b1): + '''保存最终的模型 + input: w0(mat):输入层到隐含层之间的权重 + b0(mat):输入层到隐含层之间的偏置 + w1(mat):隐含层到输出层之间的权重 + b1(mat):隐含层到输出层之间的偏置 + output: + ''' + def write_file(file_name, source): + f = open(file_name, "w") + m, n = np.shape(source) + for i in xrange(m): + tmp = [] + for j in xrange(n): + tmp.append(str(source[i, j])) + f.write("\t".join(tmp) + "\n") + f.close() + + write_file("weight_w0", w0) + write_file("weight_w1", w1) + write_file("weight_b0", b0) + write_file("weight_b1", b1) + +def err_rate(label, pre): + '''计算训练样本上的错误率 + input: label(mat):训练样本的标签 + pre(mat):训练样本的预测值 + output: rate[0,0](float):错误率 + ''' + m = np.shape(label)[0] + err = 0.0 + for i in xrange(m): + if label[i, 0] != pre[i, 0]: + err += 1 + rate = err / m + return rate + +if __name__ == "__main__": + # 1、导入数据 + print ("--------- 1.load data ------------") + feature, label, n_class = load_data("data.txt") + # 2、训练网络模型 + print ("--------- 2.training ------------") + w0, w1, b0, b1 = bp_train(feature, label, 20, 1000, 0.1, n_class) + # 3、保存最终的模型 + print ("--------- 3.save model ------------") + save_model(w0, w1, b0, b1) + # 4、得到最终的预测结果 + print ("--------- 4.get prediction ------------") + result = get_predict(feature, w0, w1, b0, b1) + print ("训练准确性为：", (1 - err_rate(np.argmax(label, axis=1), np.argmax(result, axis=1)))) + From 983edee9a0fec69d5b5a7f6ad42368910a54e2b4 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 26 Mar 2018 15:32:58 +0800 Subject: [PATCH 03/10] 1 --- Chapter_6 BP/bp_test.py | 10 +++---- Chapter_6 BP/bp_train.py | 61 +++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/Chapter_6 BP/bp_test.py b/Chapter_6 BP/bp_test.py index 99ba9cd..9e3c868 100644 --- a/Chapter_6 BP/bp_test.py +++ b/Chapter_6 BP/bp_test.py @@ -16,7 +16,7 @@ def load_data(file_name): for line in f.readlines(): feature_tmp = [] lines = line.strip().split("\t") - for i in xrange(len(lines)): + for i in range(len(lines)): feature_tmp.append(float(lines[i])) feature_data.append(feature_tmp) f.close() # 关闭文件 @@ -29,15 +29,15 @@ def generate_data(): data = np.mat(np.zeros((20000, 2))) m = np.shape(data)[0] x = np.mat(np.random.rand(20000, 2)) - for i in xrange(m): + for i in range(m): data[i, 0] = x[i, 0] * 9 - 4.5 data[i, 1] = x[i, 1] * 9 - 4.5 # 2、将数据点保存到文件“test_data”中 f = open("test_data", "w") m,n = np.shape(data) - for i in xrange(m): + for i in range(m): tmp =[] - for j in xrange(n): + for j in range(n): tmp.append(str(data[i,j])) f.write("\t".join(tmp) + "\n") f.close() @@ -78,7 +78,7 @@ def save_predict(file_name, pre): f = open(file_name, "w") m = np.shape(pre)[0] result = [] - for i in xrange(m): + for i in range(m): result.append(str(pre[i, 0])) f.write("\n".join(result)) f.close() diff --git a/Chapter_6 BP/bp_train.py b/Chapter_6 BP/bp_train.py index 3143e53..3b4b4f7 100644 --- a/Chapter_6 BP/bp_train.py +++ b/Chapter_6 BP/bp_train.py @@ -5,6 +5,8 @@ ''' import numpy as np from math import sqrt +import pandas as pd +import matplotlib.pyplot as plt def load_data(file_name): '''导入数据 @@ -19,8 +21,8 @@ def load_data(file_name): label_tmp = [] for line in f.readlines(): feature_tmp = [] - lines = line.strip().split("\t") - for i in xrange(len(lines) - 1): + lines = line.strip().split("\t") #Python strip() 方法用于移除字符串头尾指定的字符（默认为空格）。 + for i in range(len(lines) - 1): feature_tmp.append(float(lines[i])) label_tmp.append(int(lines[-1])) feature_data.append(feature_tmp) @@ -30,9 +32,9 @@ def load_data(file_name): m = len(label_tmp) n_class = len(set(label_tmp)) # 得到类别的个数 - label_data = np.mat(np.zeros((m, n_class))) - for i in xrange(m): - label_data[i, label_tmp[i]] = 1 + label_data = np.mat(np.zeros((m, n_class))) #行是数据总数，列是标签种类数 + for i in range(m): + label_data[i, label_tmp[i]] = 1 # 属于哪一类，就在那个类上把值赋为1 return np.mat(feature_data), label_data, n_class @@ -48,10 +50,10 @@ def partial_sig(x): input: x(mat/float):自变量，可以是矩阵或者是任意实数 output: out(mat/float):Sigmoid导函数的值 ''' - m, n = np.shape(x) + m, n = np.shape(x) #m为行数，n为列数 out = np.mat(np.zeros((m, n))) - for i in xrange(m): - for j in xrange(n): + for i in range(m): + for j in range(n): out[i, j] = sig(x[i, j]) * (1 - sig(x[i, j])) return out @@ -64,7 +66,11 @@ def hidden_in(feature, w0, b0): ''' m = np.shape(feature)[0] hidden_in = feature * w0 - for i in xrange(m): + + for i in range(m): + #print(w0) + #print(hidden_in) + #print(hidden_in[i, ]) hidden_in[i, ] += b0 return hidden_in @@ -85,7 +91,7 @@ def predict_in(hidden_out, w1, b1): ''' m = np.shape(hidden_out)[0] predict_in = hidden_out * w1 - for i in xrange(m): + for i in range(m): predict_in[i, ] += b1 return predict_in @@ -167,8 +173,8 @@ def get_cost(cost): m,n = np.shape(cost) cost_sum = 0.0 - for i in xrange(m): - for j in xrange(n): + for i in range(m): + for j in range(n): cost_sum += cost[i,j] * cost[i,j] return cost_sum / m @@ -194,9 +200,9 @@ def save_model(w0, w1, b0, b1): def write_file(file_name, source): f = open(file_name, "w") m, n = np.shape(source) - for i in xrange(m): + for i in range(m): tmp = [] - for j in xrange(n): + for j in range(n): tmp.append(str(source[i, j])) f.write("\t".join(tmp) + "\n") f.close() @@ -212,9 +218,9 @@ def err_rate(label, pre): pre(mat):训练样本的预测值 output: rate[0,0](float):错误率 ''' - m = np.shape(label)[0] + m = np.shape(label)[0] #行数 err = 0.0 - for i in xrange(m): + for i in range(m): if label[i, 0] != pre[i, 0]: err += 1 rate = err / m @@ -224,6 +230,7 @@ def err_rate(label, pre): # 1、导入数据 print ("--------- 1.load data ------------") feature, label, n_class = load_data("data.txt") + # 2、训练网络模型 print ("--------- 2.training ------------") w0, w1, b0, b1 = bp_train(feature, label, 20, 1000, 0.1, n_class) @@ -234,4 +241,26 @@ def err_rate(label, pre): print ("--------- 4.get prediction ------------") result = get_predict(feature, w0, w1, b0, b1) print ("训练准确性为：", (1 - err_rate(np.argmax(label, axis=1), np.argmax(result, axis=1)))) + #np.argmax(label, axis=1) 返回的是同一行中最大值的列数。 + + + +''' + --------- 1.load data ------------ +--------- 2.training ------------ + -------- iter: 0 ,cost: 0.383725119032 + -------- iter: 100 ,cost: 0.0281588166387 + -------- iter: 200 ,cost: 0.0279331547562 + -------- iter: 300 ,cost: 0.0249447828392 + -------- iter: 400 ,cost: 0.012409386771 + -------- iter: 500 ,cost: 0.0117987327196 + -------- iter: 600 ,cost: 0.0115194123029 + -------- iter: 700 ,cost: 0.0110088095243 + -------- iter: 800 ,cost: 0.0105985235697 + -------- iter: 900 ,cost: 0.00829191991913 + -------- iter: 1000 ,cost: 0.00846503398547 +--------- 3.save model ------------ +--------- 4.get prediction ------------ +训练准确性为： 0.9925 +''' From ba7320345ef26f0691dda91505e299b721fac6a0 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Tue, 27 Mar 2018 16:02:02 +0800 Subject: [PATCH 04/10] 1 --- Chapter_10 KMeans/KMeans.py | 252 ++++++++++++++++++---------------- Chapter_10 KMeans/KMeanspp.py | 156 ++++++++++----------- 2 files changed, 208 insertions(+), 200 deletions(-) diff --git a/Chapter_10 KMeans/KMeans.py b/Chapter_10 KMeans/KMeans.py index d0ab004..cc2f8ec 100644 --- a/Chapter_10 KMeans/KMeans.py +++ b/Chapter_10 KMeans/KMeans.py @@ -1,122 +1,130 @@ -# coding:UTF-8 -''' -Date:20160923 -@author: zhaozhiyong -''' -import numpy as np - -def load_data(file_path): - '''导入数据 - input: file_path(string):文件的存储位置 - output: data(mat):数据 - ''' - f = open(file_path) - data = [] - for line in f.readlines(): - row = [] # 记录每一行 - lines = line.strip().split("\t") - for x in lines: - row.append(float(x)) # 将文本中的特征转换成浮点数 - data.append(row) - f.close() - return np.mat(data) - -def distance(vecA, vecB): - '''计算vecA与vecB之间的欧式距离的平方 - input: vecA(mat)A点坐标 - vecB(mat)B点坐标 - output: dist[0, 0](float)A点与B点距离的平方 - ''' - dist = (vecA - vecB) * (vecA - vecB).T - return dist[0, 0] - -def randCent(data, k): - '''随机初始化聚类中心 - input: data(mat):训练数据 - k(int):类别个数 - output: centroids(mat):聚类中心 - ''' - n = np.shape(data)[1] # 属性的个数 - centroids = np.mat(np.zeros((k, n))) # 初始化k个聚类中心 - for j in xrange(n): # 初始化聚类中心每一维的坐标 - minJ = np.min(data[:, j]) - rangeJ = np.max(data[:, j]) - minJ - # 在最大值和最小值之间随机初始化 - centroids[:, j] = minJ * np.mat(np.ones((k , 1))) \ - + np.random.rand(k, 1) * rangeJ - return centroids - -def kmeans(data, k, centroids): - '''根据KMeans算法求解聚类中心 - input: data(mat):训练数据 - k(int):类别个数 - centroids(mat):随机初始化的聚类中心 - output: centroids(mat):训练完成的聚类中心 - subCenter(mat):每一个样本所属的类别 - ''' - m, n = np.shape(data) # m：样本的个数，n：特征的维度 - subCenter = np.mat(np.zeros((m, 2))) # 初始化每一个样本所属的类别 - change = True # 判断是否需要重新计算聚类中心 - while change == True: - change = False # 重置 - for i in xrange(m): - minDist = np.inf # 设置样本与聚类中心之间的最小的距离，初始值为正无穷 - minIndex = 0 # 所属的类别 - for j in xrange(k): - # 计算i和每个聚类中心之间的距离 - dist = distance(data[i, ], centroids[j, ]) - if dist < minDist: - minDist = dist - minIndex = j - # 判断是否需要改变 - if subCenter[i, 0] <> minIndex: # 需要改变 - change = True - subCenter[i, ] = np.mat([minIndex, minDist]) - # 重新计算聚类中心 - for j in xrange(k): - sum_all = np.mat(np.zeros((1, n))) - r = 0 # 每个类别中的样本的个数 - for i in xrange(m): - if subCenter[i, 0] == j: # 计算第j个类别 - sum_all += data[i, ] - r += 1 - for z in xrange(n): - try: - centroids[j, z] = sum_all[0, z] / r - except: - print " r is zero" - return subCenter - -def save_result(file_name, source): - '''保存source中的结果到file_name文件中 - input: file_name(string):文件名 - source(mat):需要保存的数据 - output: - ''' - m, n = np.shape(source) - f = open(file_name, "w") - for i in xrange(m): - tmp = [] - for j in xrange(n): - tmp.append(str(source[i, j])) - f.write("\t".join(tmp) + "\n") - f.close() - -if __name__ == "__main__": - k = 4 # 聚类中心的个数 - file_path = "data.txt" - # 1、导入数据 - print "---------- 1.load data ------------" - data = load_data(file_path) - # 2、随机初始化k个聚类中心 - print "---------- 2.random center ------------" - centroids = randCent(data, k) - # 3、聚类计算 - print "---------- 3.kmeans ------------" - subCenter = kmeans(data, k, centroids) - # 4、保存所属的类别文件 - print "---------- 4.save subCenter ------------" - save_result("sub", subCenter) - # 5、保存聚类中心 - print "---------- 5.save centroids ------------" - save_result("center", centroids) +# coding:UTF-8 +''' +Date:20160923 +@author: zhaozhiyong +''' +import numpy as np + +def load_data(file_path): + '''导入数据 + input: file_path(string):文件的存储位置 + output: data(mat):数据 + ''' + f = open(file_path) + data = [] + for line in f.readlines(): + row = [] # 记录每一行 + lines = line.strip().split("\t") + for x in lines: + row.append(float(x)) # 将文本中的特征转换成浮点数 + data.append(row) + f.close() + return np.mat(data) + +def distance(vecA, vecB): + '''计算vecA与vecB之间的欧式距离的平方 + input: vecA(mat)A点坐标 + vecB(mat)B点坐标 + output: dist[0, 0](float)A点与B点距离的平方 + ''' + dist = (vecA - vecB) * (vecA - vecB).T + return dist[0, 0] + +def randCent(data, k): + '''随机初始化聚类中心 + input: data(mat):训练数据 + k(int):类别个数 + output: centroids(mat):聚类中心 + ''' + n = np.shape(data)[1] # 属性的个数 + centroids = np.mat(np.zeros((k, n))) # 初始化k个聚类中心 + for j in range(n): # 初始化聚类中心每一维的坐标 + #print(data[:, j]) + + minJ = np.min(data[:, j]) + #minJ = np.min(data[None, j]) + + rangeJ = np.max(data[:, j]) - minJ + # 在最大值和最小值之间随机初始化 + centroids[:, j] = minJ * np.mat(np.ones((k , 1))) \ + + np.random.rand(k, 1) * rangeJ + +#numpy.random.randn(d0, d1, …, dn)是从标准正态分布中返回一个或多个样本值。 +#numpy.random.rand(d0, d1, …, dn)的随机样本位于[0, 1)中。 + + return centroids + +def kmeans(data, k, centroids): + '''根据KMeans算法求解聚类中心 + input: data(mat):训练数据 + k(int):类别个数 + centroids(mat):随机初始化的聚类中心 + output: centroids(mat):训练完成的聚类中心 + subCenter(mat):每一个样本所属的类别 + ''' + m, n = np.shape(data) # m：样本的个数，n：特征的维度 + subCenter = np.mat(np.zeros((m, 2))) # 初始化每一个样本所属的类别 （第一列：minIndex 第二列：minDist） + change = True # 判断是否需要重新计算聚类中心 + while change == True: + change = False # 重置 + for i in range(m): + minDist = np.inf # 设置样本与聚类中心之间的最小的距离，初始值为正无穷 + minIndex = 0 # 所属的类别 + for j in range(k): + # 计算i和每个聚类中心之间的距离 + dist = distance(data[i, ], centroids[j, ]) + if dist < minDist: + minDist = dist + minIndex = j + # 判断是否需要改变 + if subCenter[i, 0] != minIndex: # 需要改变 + change = True + subCenter[i, ] = np.mat([minIndex, minDist]) #改变样本所属的类别 + # 重新计算聚类中心 + for j in range(k): + sum_all = np.mat(np.zeros((1, n))) + r = 0 # 每个类别中的样本的个数 + for i in range(m): + if subCenter[i, 0] == j: # 计算第j个类别 + sum_all += data[i, ] + r += 1 + for z in range(n): + try: + centroids[j, z] = sum_all[0, z] / r #将同一个类别的样本数据相加 求平均值得到该类别聚类中心 + except: + print (" r is zero") + return subCenter + +def save_result(file_name, source): + '''保存source中的结果到file_name文件中 + input: file_name(string):文件名 + source(mat):需要保存的数据 + output: + ''' + m, n = np.shape(source) + f = open(file_name, "w") + for i in range(m): + tmp = [] + for j in range(n): + tmp.append(str(source[i, j])) + f.write("\t".join(tmp) + "\n") + f.close() + +if __name__ == "__main__": + k = 4 # 聚类中心的个数 + file_path = "data.txt" + # 1、导入数据 + print ("---------- 1.load data ------------") + data = load_data(file_path) + # 2、随机初始化k个聚类中心 + print ("---------- 2.random center ------------") + centroids = randCent(data, k) + # 3、聚类计算 + print ("---------- 3.kmeans ------------") + subCenter = kmeans(data, k, centroids) + # 4、保存所属的类别文件 + print ("---------- 4.save subCenter ------------") + save_result("sub", subCenter) + # 5、保存聚类中心 + print ("---------- 5.save centroids ------------") + save_result("center", centroids) diff --git a/Chapter_10 KMeans/KMeanspp.py b/Chapter_10 KMeans/KMeanspp.py index 4ded298..115ad01 100644 --- a/Chapter_10 KMeans/KMeanspp.py +++ b/Chapter_10 KMeans/KMeanspp.py @@ -1,78 +1,78 @@ -# coding:UTF-8 -''' -Date:20160923 -@author: zhaozhiyong -''' - -import numpy as np -from random import random -from KMeans import load_data, kmeans, distance, save_result - -FLOAT_MAX = 1e100 # 设置一个较大的值作为初始化的最小的距离 - -def nearest(point, cluster_centers): - '''计算point和cluster_centers之间的最小距离 - input: point(mat):当前的样本点 - cluster_centers(mat):当前已经初始化的聚类中心 - output: min_dist(float):点point和当前的聚类中心之间的最短距离 - ''' - min_dist = FLOAT_MAX - m = np.shape(cluster_centers)[0] # 当前已经初始化的聚类中心的个数 - for i in xrange(m): - # 计算point与每个聚类中心之间的距离 - d = distance(point, cluster_centers[i, ]) - # 选择最短距离 - if min_dist > d: - min_dist = d - return min_dist - -def get_centroids(points, k): - '''KMeans++的初始化聚类中心的方法 - input: points(mat):样本 - k(int):聚类中心的个数 - output: cluster_centers(mat):初始化后的聚类中心 - ''' - m, n = np.shape(points) - cluster_centers = np.mat(np.zeros((k , n))) - # 1、随机选择一个样本点为第一个聚类中心 - index = np.random.randint(0, m) - cluster_centers[0, ] = np.copy(points[index, ]) - # 2、初始化一个距离的序列 - d = [0.0 for _ in xrange(m)] - - for i in xrange(1, k): - sum_all = 0 - for j in xrange(m): - # 3、对每一个样本找到最近的聚类中心点 - d[j] = nearest(points[j, ], cluster_centers[0:i, ]) - # 4、将所有的最短距离相加 - sum_all += d[j] - # 5、取得sum_all之间的随机值 - sum_all *= random() - # 6、获得距离最远的样本点作为聚类中心点 - for j, di in enumerate(d): - sum_all -= di - if sum_all > 0: - continue - cluster_centers[i] = np.copy(points[j, ]) - break - return cluster_centers - -if __name__ == "__main__": - k = 4#聚类中心的个数 - file_path = "data.txt" - # 1、导入数据 - print "---------- 1.load data ------------" - data = load_data(file_path) - # 2、KMeans++的聚类中心初始化方法 - print "---------- 2.K-Means++ generate centers ------------" - centroids = get_centroids(data, k) - # 3、聚类计算 - print "---------- 3.kmeans ------------" - subCenter = kmeans(data, k, centroids) - # 4、保存所属的类别文件 - print "---------- 4.save subCenter ------------" - save_result("sub_pp", subCenter) - # 5、保存聚类中心 - print "---------- 5.save centroids ------------" - save_result("center_pp", centroids) +# coding:UTF-8 +''' +Date:20160923 +@author: zhaozhiyong +''' + +import numpy as np +from random import random +from KMeans import load_data, kmeans, distance, save_result + +FLOAT_MAX = 1e100 # 设置一个较大的值作为初始化的最小的距离 10的100次方 + +def nearest(point, cluster_centers): + '''计算point和cluster_centers之间的最小距离 + input: point(mat):当前的样本点 + cluster_centers(mat):当前已经初始化的聚类中心 + output: min_dist(float):点point和当前的聚类中心之间的最短距离 + ''' + min_dist = FLOAT_MAX + m = np.shape(cluster_centers)[0] # 当前已经初始化的聚类中心的个数 + for i in range(m): + # 计算point与每个聚类中心之间的距离 + d = distance(point, cluster_centers[i, ]) + # 选择最短距离 + if min_dist > d: + min_dist = d + return min_dist + +def get_centroids(points, k): + '''KMeans++的初始化聚类中心的方法 + input: points(mat):样本 + k(int):聚类中心的个数 + output: cluster_centers(mat):初始化后的聚类中心 + ''' + m, n = np.shape(points) + cluster_centers = np.mat(np.zeros((k , n))) + # 1、随机选择一个样本点为第一个聚类中心 + index = np.random.randint(0, m) + cluster_centers[0, ] = np.copy(points[index, ]) + # 2、初始化一个距离的序列 + d = [0.0 for _ in range(m)] + + for i in range(1, k): + sum_all = 0 + for j in range(m): + # 3、对每一个样本找到最近的聚类中心点 + d[j] = nearest(points[j, ], cluster_centers[0:i, ]) + # 4、将所有的最短距离相加 + sum_all += d[j] + # 5、取得sum_all之间的随机值 + sum_all *= random() + # 6、获得距离最远的样本点作为聚类中心点 + for j, di in enumerate(d): + sum_all -= di + if sum_all > 0: + continue + cluster_centers[i] = np.copy(points[j, ]) + break + return cluster_centers + +if __name__ == "__main__": + k = 4#聚类中心的个数 + file_path = "data.txt" + # 1、导入数据 + print ("---------- 1.load data ------------") + data = load_data(file_path) + # 2、KMeans++的聚类中心初始化方法 + print ("---------- 2.K-Means++ generate centers ------------") + centroids = get_centroids(data, k) + # 3、聚类计算 + print ("---------- 3.kmeans ------------") + subCenter = kmeans(data, k, centroids) + # 4、保存所属的类别文件 + print ("---------- 4.save subCenter ------------") + save_result("sub_pp", subCenter) + # 5、保存聚类中心 + print ("---------- 5.save centroids ------------") + save_result("center_pp", centroids) From 3745519d2e9d6fc07277c746136d95f72f631389 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Tue, 27 Mar 2018 19:14:52 +0800 Subject: [PATCH 05/10] 1 --- Chapter_4 SVM/svm.py | 529 ++++++++++++++++++++++--------------------- 1 file changed, 265 insertions(+), 264 deletions(-) diff --git a/Chapter_4 SVM/svm.py b/Chapter_4 SVM/svm.py index 6f0b7ad..588eeb7 100644 --- a/Chapter_4 SVM/svm.py +++ b/Chapter_4 SVM/svm.py @@ -1,264 +1,265 @@ -#coding:UTF-8 -import numpy as np -import cPickle as pickle - -class SVM: - def __init__(self, dataSet, labels, C, toler, kernel_option): - self.train_x = dataSet # 训练特征 - self.train_y = labels # 训练标签 - self.C = C # 惩罚参数 - self.toler = toler # 迭代的终止条件之一 - self.n_samples = np.shape(dataSet)[0] # 训练样本的个数 - self.alphas = np.mat(np.zeros((self.n_samples, 1))) # 拉格朗日乘子 - self.b = 0 - self.error_tmp = np.mat(np.zeros((self.n_samples, 2))) # 保存E的缓存 - self.kernel_opt = kernel_option # 选用的核函数及其参数 - self.kernel_mat = calc_kernel(self.train_x, self.kernel_opt) # 核函数的输出 - -def cal_kernel_value(train_x, train_x_i, kernel_option): - '''样本之间的核函数的值 - input: train_x(mat):训练样本 - train_x_i(mat):第i个训练样本 - kernel_option(tuple):核函数的类型以及参数 - output: kernel_value(mat):样本之间的核函数的值 - - ''' - kernel_type = kernel_option[0] # 核函数的类型，分为rbf和其他 - m = np.shape(train_x)[0] # 样本的个数 - - kernel_value = np.mat(np.zeros((m, 1))) - - if kernel_type == 'rbf': # rbf核函数 - sigma = kernel_option[1] - if sigma == 0: - sigma = 1.0 - for i in xrange(m): - diff = train_x[i, :] - train_x_i - kernel_value[i] = np.exp(diff * diff.T / (-2.0 * sigma**2)) - else: # 不使用核函数 - kernel_value = train_x * train_x_i.T - return kernel_value - - -def calc_kernel(train_x, kernel_option): - '''计算核函数矩阵 - input: train_x(mat):训练样本的特征值 - kernel_option(tuple):核函数的类型以及参数 - output: kernel_matrix(mat):样本的核函数的值 - ''' - m = np.shape(train_x)[0] # 样本的个数 - kernel_matrix = np.mat(np.zeros((m, m))) # 初始化样本之间的核函数值 - for i in xrange(m): - kernel_matrix[:, i] = cal_kernel_value(train_x, train_x[i, :], kernel_option) - return kernel_matrix - -def cal_error(svm, alpha_k): - '''误差值的计算 - input: svm:SVM模型 - alpha_k(int):选择出的变量 - output: error_k(float):误差值 - ''' - output_k = float(np.multiply(svm.alphas, svm.train_y).T * svm.kernel_mat[:, alpha_k] + svm.b) - error_k = output_k - float(svm.train_y[alpha_k]) - return error_k - - -def update_error_tmp(svm, alpha_k): - '''重新计算误差值 - input: svm:SVM模型 - alpha_k(int):选择出的变量 - output: 对应误差值 - ''' - error = cal_error(svm, alpha_k) - svm.error_tmp[alpha_k] = [1, error] - -def select_second_sample_j(svm, alpha_i, error_i): - '''选择第二个样本 - input: svm:SVM模型 - alpha_i(int):选择出的第一个变量 - error_i(float):E_i - output: alpha_j(int):选择出的第二个变量 - error_j(float):E_j - ''' - # 标记为已被优化 - svm.error_tmp[alpha_i] = [1, error_i] - candidateAlphaList = np.nonzero(svm.error_tmp[:, 0].A)[0] - - maxStep = 0 - alpha_j = 0 - error_j = 0 - - if len(candidateAlphaList) > 1: - for alpha_k in candidateAlphaList: - if alpha_k == alpha_i: - continue - error_k = cal_error(svm, alpha_k) - if abs(error_k - error_i) > maxStep: - maxStep = abs(error_k - error_i) - alpha_j = alpha_k - error_j = error_k - else: # 随机选择 - alpha_j = alpha_i - while alpha_j == alpha_i: - alpha_j = int(np.random.uniform(0, svm.n_samples)) - error_j = cal_error(svm, alpha_j) - - return alpha_j, error_j - -def choose_and_update(svm, alpha_i): - '''判断和选择两个alpha进行更新 - input: svm:SVM模型 - alpha_i(int):选择出的第一个变量 - ''' - error_i = cal_error(svm, alpha_i) # 计算第一个样本的E_i - - # 判断选择出的第一个变量是否违反了KKT条件 - if (svm.train_y[alpha_i] * error_i < -svm.toler) and (svm.alphas[alpha_i] < svm.C) or\ - (svm.train_y[alpha_i] * error_i > svm.toler) and (svm.alphas[alpha_i] > 0): - - # 1、选择第二个变量 - alpha_j, error_j = select_second_sample_j(svm, alpha_i, error_i) - alpha_i_old = svm.alphas[alpha_i].copy() - alpha_j_old = svm.alphas[alpha_j].copy() - - # 2、计算上下界 - if svm.train_y[alpha_i] != svm.train_y[alpha_j]: - L = max(0, svm.alphas[alpha_j] - svm.alphas[alpha_i]) - H = min(svm.C, svm.C + svm.alphas[alpha_j] - svm.alphas[alpha_i]) - else: - L = max(0, svm.alphas[alpha_j] + svm.alphas[alpha_i] - svm.C) - H = min(svm.C, svm.alphas[alpha_j] + svm.alphas[alpha_i]) - if L == H: - return 0 - - # 3、计算eta - eta = 2.0 * svm.kernel_mat[alpha_i, alpha_j] - svm.kernel_mat[alpha_i, alpha_i] \ - - svm.kernel_mat[alpha_j, alpha_j] - if eta >= 0: - return 0 - - # 4、更新alpha_j - svm.alphas[alpha_j] -= svm.train_y[alpha_j] * (error_i - error_j) / eta - - # 5、确定最终的alpha_j - if svm.alphas[alpha_j] > H: - svm.alphas[alpha_j] = H - if svm.alphas[alpha_j] < L: - svm.alphas[alpha_j] = L - - # 6、判断是否结束 - if abs(alpha_j_old - svm.alphas[alpha_j]) < 0.00001: - update_error_tmp(svm, alpha_j) - return 0 - - # 7、更新alpha_i - svm.alphas[alpha_i] += svm.train_y[alpha_i] * svm.train_y[alpha_j] \ - * (alpha_j_old - svm.alphas[alpha_j]) - - # 8、更新b - b1 = svm.b - error_i - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \ - * svm.kernel_mat[alpha_i, alpha_i] \ - - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \ - * svm.kernel_mat[alpha_i, alpha_j] - b2 = svm.b - error_j - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \ - * svm.kernel_mat[alpha_i, alpha_j] \ - - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \ - * svm.kernel_mat[alpha_j, alpha_j] - if (0 < svm.alphas[alpha_i]) and (svm.alphas[alpha_i] < svm.C): - svm.b = b1 - elif (0 < svm.alphas[alpha_j]) and (svm.alphas[alpha_j] < svm.C): - svm.b = b2 - else: - svm.b = (b1 + b2) / 2.0 - - # 9、更新error - update_error_tmp(svm, alpha_j) - update_error_tmp(svm, alpha_i) - - return 1 - else: - return 0 - -def SVM_training(train_x, train_y, C, toler, max_iter, kernel_option = ('rbf', 0.431029)): - '''SVM的训练 - input: train_x(mat):训练数据的特征 - train_y(mat):训练数据的标签 - C(float):惩罚系数 - toler(float):迭代的终止条件之一 - max_iter(int):最大迭代次数 - kerner_option(tuple):核函数的类型及其参数 - output: svm模型 - ''' - # 1、初始化SVM分类器 - svm = SVM(train_x, train_y, C, toler, kernel_option) - - # 2、开始训练 - entireSet = True - alpha_pairs_changed = 0 - iteration = 0 - - while (iteration < max_iter) and ((alpha_pairs_changed > 0) or entireSet): - print "\t iterration: ", iteration - alpha_pairs_changed = 0 - - if entireSet: - # 对所有的样本 - for x in xrange(svm.n_samples): - alpha_pairs_changed += choose_and_update(svm, x) - iteration += 1 - else: - # 非边界样本 - bound_samples = [] - for i in xrange(svm.n_samples): - if svm.alphas[i,0] > 0 and svm.alphas[i,0] < svm.C: - bound_samples.append(i) - for x in bound_samples: - alpha_pairs_changed += choose_and_update(svm, x) - iteration += 1 - - # 在所有样本和非边界样本之间交替 - if entireSet: - entireSet = False - elif alpha_pairs_changed == 0: - entireSet = True - - return svm - -def svm_predict(svm, test_sample_x): - '''利用SVM模型对每一个样本进行预测 - input: svm:SVM模型 - test_sample_x(mat):样本 - output: predict(float):对样本的预测 - ''' - # 1、计算核函数矩阵 - kernel_value = cal_kernel_value(svm.train_x, test_sample_x, svm.kernel_opt) - # 2、计算预测值 - predict = kernel_value.T * np.multiply(svm.train_y, svm.alphas) + svm.b - return predict - -def cal_accuracy(svm, test_x, test_y): - '''计算预测的准确性 - input: svm:SVM模型 - test_x(mat):测试的特征 - test_y(mat):测试的标签 - output: accuracy(float):预测的准确性 - ''' - n_samples = np.shape(test_x)[0] # 样本的个数 - correct = 0.0 - for i in xrange(n_samples): - # 对每一个样本得到预测值 - predict=svm_predict(svm, test_x[i, :]) - # 判断每一个样本的预测值与真实值是否一致 - if np.sign(predict) == np.sign(test_y[i]): - correct += 1 - accuracy = correct / n_samples - return accuracy - -def save_svm_model(svm_model, model_file): - '''保存SVM模型 - input: svm_model:SVM模型 - model_file(string):SVM模型需要保存到的文件 - ''' - with open(model_file, 'w') as f: - pickle.dump(svm_model, f) - +#coding:UTF-8 +import numpy as np +#import cPickle as pickle #cPickle是python2中内容 +import pickle as pickle + +class SVM: + def __init__(self, dataSet, labels, C, toler, kernel_option): + self.train_x = dataSet # 训练特征 + self.train_y = labels # 训练标签 + self.C = C # 惩罚参数 + self.toler = toler # 迭代的终止条件之一 + self.n_samples = np.shape(dataSet)[0] # 训练样本的个数 + self.alphas = np.mat(np.zeros((self.n_samples, 1))) # 拉格朗日乘子 + self.b = 0 + self.error_tmp = np.mat(np.zeros((self.n_samples, 2))) # 保存E的缓存 + self.kernel_opt = kernel_option # 选用的核函数及其参数 + self.kernel_mat = calc_kernel(self.train_x, self.kernel_opt) # 核函数的输出 + +def cal_kernel_value(train_x, train_x_i, kernel_option): + '''样本之间的核函数的值 + input: train_x(mat):训练样本 + train_x_i(mat):第i个训练样本 + kernel_option(tuple):核函数的类型以及参数 + output: kernel_value(mat):样本之间的核函数的值 + + ''' + kernel_type = kernel_option[0] # 核函数的类型，分为rbf和其他 + m = np.shape(train_x)[0] # 样本的个数 + + kernel_value = np.mat(np.zeros((m, 1))) + + if kernel_type == 'rbf': # rbf核函数 + sigma = kernel_option[1] + if sigma == 0: + sigma = 1.0 + for i in range(m): + diff = train_x[i, :] - train_x_i + kernel_value[i] = np.exp(diff * diff.T / (-2.0 * sigma**2)) + else: # 不使用核函数 + kernel_value = train_x * train_x_i.T + return kernel_value + + +def calc_kernel(train_x, kernel_option): + '''计算核函数矩阵 + input: train_x(mat):训练样本的特征值 + kernel_option(tuple):核函数的类型以及参数 + output: kernel_matrix(mat):样本的核函数的值 + ''' + m = np.shape(train_x)[0] # 样本的个数 + kernel_matrix = np.mat(np.zeros((m, m))) # 初始化样本之间的核函数值 + for i in range(m): + kernel_matrix[:, i] = cal_kernel_value(train_x, train_x[i, :], kernel_option) + return kernel_matrix + +def cal_error(svm, alpha_k): + '''误差值的计算 + input: svm:SVM模型 + alpha_k(int):选择出的变量 + output: error_k(float):误差值 + ''' + output_k = float(np.multiply(svm.alphas, svm.train_y).T * svm.kernel_mat[:, alpha_k] + svm.b) + error_k = output_k - float(svm.train_y[alpha_k]) + return error_k + + +def update_error_tmp(svm, alpha_k): + '''重新计算误差值 + input: svm:SVM模型 + alpha_k(int):选择出的变量 + output: 对应误差值 + ''' + error = cal_error(svm, alpha_k) + svm.error_tmp[alpha_k] = [1, error] + +def select_second_sample_j(svm, alpha_i, error_i): + '''选择第二个样本 + input: svm:SVM模型 + alpha_i(int):选择出的第一个变量 + error_i(float):E_i + output: alpha_j(int):选择出的第二个变量 + error_j(float):E_j + ''' + # 标记为已被优化 + svm.error_tmp[alpha_i] = [1, error_i] + candidateAlphaList = np.nonzero(svm.error_tmp[:, 0].A)[0] + + maxStep = 0 + alpha_j = 0 + error_j = 0 + + if len(candidateAlphaList) > 1: + for alpha_k in candidateAlphaList: + if alpha_k == alpha_i: + continue + error_k = cal_error(svm, alpha_k) + if abs(error_k - error_i) > maxStep: + maxStep = abs(error_k - error_i) + alpha_j = alpha_k + error_j = error_k + else: # 随机选择 + alpha_j = alpha_i + while alpha_j == alpha_i: + alpha_j = int(np.random.uniform(0, svm.n_samples)) + error_j = cal_error(svm, alpha_j) + + return alpha_j, error_j + +def choose_and_update(svm, alpha_i): + '''判断和选择两个alpha进行更新 + input: svm:SVM模型 + alpha_i(int):选择出的第一个变量 + ''' + error_i = cal_error(svm, alpha_i) # 计算第一个样本的E_i + + # 判断选择出的第一个变量是否违反了KKT条件 + if (svm.train_y[alpha_i] * error_i < -svm.toler) and (svm.alphas[alpha_i] < svm.C) or\ + (svm.train_y[alpha_i] * error_i > svm.toler) and (svm.alphas[alpha_i] > 0): + + # 1、选择第二个变量 + alpha_j, error_j = select_second_sample_j(svm, alpha_i, error_i) + alpha_i_old = svm.alphas[alpha_i].copy() + alpha_j_old = svm.alphas[alpha_j].copy() + + # 2、计算上下界 + if svm.train_y[alpha_i] != svm.train_y[alpha_j]: + L = max(0, svm.alphas[alpha_j] - svm.alphas[alpha_i]) + H = min(svm.C, svm.C + svm.alphas[alpha_j] - svm.alphas[alpha_i]) + else: + L = max(0, svm.alphas[alpha_j] + svm.alphas[alpha_i] - svm.C) + H = min(svm.C, svm.alphas[alpha_j] + svm.alphas[alpha_i]) + if L == H: + return 0 + + # 3、计算eta + eta = 2.0 * svm.kernel_mat[alpha_i, alpha_j] - svm.kernel_mat[alpha_i, alpha_i] \ + - svm.kernel_mat[alpha_j, alpha_j] + if eta >= 0: + return 0 + + # 4、更新alpha_j + svm.alphas[alpha_j] -= svm.train_y[alpha_j] * (error_i - error_j) / eta + + # 5、确定最终的alpha_j + if svm.alphas[alpha_j] > H: + svm.alphas[alpha_j] = H + if svm.alphas[alpha_j] < L: + svm.alphas[alpha_j] = L + + # 6、判断是否结束 + if abs(alpha_j_old - svm.alphas[alpha_j]) < 0.00001: + update_error_tmp(svm, alpha_j) + return 0 + + # 7、更新alpha_i + svm.alphas[alpha_i] += svm.train_y[alpha_i] * svm.train_y[alpha_j] \ + * (alpha_j_old - svm.alphas[alpha_j]) + + # 8、更新b + b1 = svm.b - error_i - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \ + * svm.kernel_mat[alpha_i, alpha_i] \ + - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \ + * svm.kernel_mat[alpha_i, alpha_j] + b2 = svm.b - error_j - svm.train_y[alpha_i] * (svm.alphas[alpha_i] - alpha_i_old) \ + * svm.kernel_mat[alpha_i, alpha_j] \ + - svm.train_y[alpha_j] * (svm.alphas[alpha_j] - alpha_j_old) \ + * svm.kernel_mat[alpha_j, alpha_j] + if (0 < svm.alphas[alpha_i]) and (svm.alphas[alpha_i] < svm.C): + svm.b = b1 + elif (0 < svm.alphas[alpha_j]) and (svm.alphas[alpha_j] < svm.C): + svm.b = b2 + else: + svm.b = (b1 + b2) / 2.0 + + # 9、更新error + update_error_tmp(svm, alpha_j) + update_error_tmp(svm, alpha_i) + + return 1 + else: + return 0 + +def SVM_training(train_x, train_y, C, toler, max_iter, kernel_option = ('rbf', 0.431029)): + '''SVM的训练 + input: train_x(mat):训练数据的特征 + train_y(mat):训练数据的标签 + C(float):惩罚系数 + toler(float):迭代的终止条件之一 + max_iter(int):最大迭代次数 + kerner_option(tuple):核函数的类型及其参数 + output: svm模型 + ''' + # 1、初始化SVM分类器 + svm = SVM(train_x, train_y, C, toler, kernel_option) + + # 2、开始训练 + entireSet = True + alpha_pairs_changed = 0 + iteration = 0 + + while (iteration < max_iter) and ((alpha_pairs_changed > 0) or entireSet): + print "\t iterration: ", iteration + alpha_pairs_changed = 0 + + if entireSet: + # 对所有的样本 + for x in range(svm.n_samples): + alpha_pairs_changed += choose_and_update(svm, x) + iteration += 1 + else: + # 非边界样本 + bound_samples = [] + for i in range(svm.n_samples): + if svm.alphas[i,0] > 0 and svm.alphas[i,0] < svm.C: + bound_samples.append(i) + for x in bound_samples: + alpha_pairs_changed += choose_and_update(svm, x) + iteration += 1 + + # 在所有样本和非边界样本之间交替 + if entireSet: + entireSet = False + elif alpha_pairs_changed == 0: + entireSet = True + + return svm + +def svm_predict(svm, test_sample_x): + '''利用SVM模型对每一个样本进行预测 + input: svm:SVM模型 + test_sample_x(mat):样本 + output: predict(float):对样本的预测 + ''' + # 1、计算核函数矩阵 + kernel_value = cal_kernel_value(svm.train_x, test_sample_x, svm.kernel_opt) + # 2、计算预测值 + predict = kernel_value.T * np.multiply(svm.train_y, svm.alphas) + svm.b + return predict + +def cal_accuracy(svm, test_x, test_y): + '''计算预测的准确性 + input: svm:SVM模型 + test_x(mat):测试的特征 + test_y(mat):测试的标签 + output: accuracy(float):预测的准确性 + ''' + n_samples = np.shape(test_x)[0] # 样本的个数 + correct = 0.0 + for i in range(n_samples): + # 对每一个样本得到预测值 + predict=svm_predict(svm, test_x[i, :]) + # 判断每一个样本的预测值与真实值是否一致 + if np.sign(predict) == np.sign(test_y[i]): + correct += 1 + accuracy = correct / n_samples + return accuracy + +def save_svm_model(svm_model, model_file): + '''保存SVM模型 + input: svm_model:SVM模型 + model_file(string):SVM模型需要保存到的文件 + ''' + with open(model_file, 'w') as f: + pickle.dump(svm_model, f) + From f9677fc7124023a199e98191c5869ced7235a1d9 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Wed, 28 Mar 2018 19:08:18 +0800 Subject: [PATCH 06/10] 1 --- Chapter_1 Logistic Regression/lr_train.py | 198 +++++------ Chapter_4 SVM/svm.py | 2 +- .../random_forests_train.py | 310 +++++++++--------- Chapter_5 Random Forest/tree.py | 262 +++++++-------- Chapter_6 BP/bp_train.py | 5 +- 5 files changed, 388 insertions(+), 389 deletions(-) diff --git a/Chapter_1 Logistic Regression/lr_train.py b/Chapter_1 Logistic Regression/lr_train.py index 7a1251e..2016c4c 100644 --- a/Chapter_1 Logistic Regression/lr_train.py +++ b/Chapter_1 Logistic Regression/lr_train.py @@ -1,99 +1,99 @@ -# coding:UTF-8 -''' -Date:20160901 -@author: zhaozhiyong -''' -import numpy as np - -def load_data(file_name): - '''导入训练数据 - input: file_name(string)训练数据的位置 - output: feature_data(mat)特征 - label_data(mat)标签 - ''' - f = open(file_name) # 打开文件 - feature_data = [] - label_data = [] - for line in f.readlines(): - feature_tmp = [] - lable_tmp = [] - lines = line.strip().split("\t") - feature_tmp.append(1) # 偏置项 - for i in xrange(len(lines) - 1): - feature_tmp.append(float(lines[i])) - lable_tmp.append(float(lines[-1])) - - feature_data.append(feature_tmp) - label_data.append(lable_tmp) - f.close() # 关闭文件 - return np.mat(feature_data), np.mat(label_data) - -def sig(x): - '''Sigmoid函数 - input: x(mat):feature * w - output: sigmoid(x)(mat):Sigmoid值 - ''' - return 1.0 / (1 + np.exp(-x)) - -def lr_train_bgd(feature, label, maxCycle, alpha): - '''利用梯度下降法训练LR模型 - input: feature(mat)特征 - label(mat)标签 - maxCycle(int)最大迭代次数 - alpha(float)学习率 - output: w(mat):权重 - ''' - n = np.shape(feature)[1] # 特征个数 - w = np.mat(np.ones((n, 1))) # 初始化权重 - i = 0 - while i <= maxCycle: # 在最大迭代次数的范围内 - i += 1 # 当前的迭代次数 - h = sig(feature * w) # 计算Sigmoid值 - err = label - h - if i % 100 == 0: - print "\t---------iter=" + str(i) + \ - " , train error rate= " + str(error_rate(h, label)) - w = w + alpha * feature.T * err # 权重修正 - return w - -def error_rate(h, label): - '''计算当前的损失函数值 - input: h(mat):预测值 - label(mat):实际值 - output: err/m(float):错误率 - ''' - m = np.shape(h)[0] - - sum_err = 0.0 - for i in xrange(m): - if h[i, 0] > 0 and (1 - h[i, 0]) > 0: - sum_err -= (label[i,0] * np.log(h[i,0]) + \ - (1-label[i,0]) * np.log(1-h[i,0])) - else: - sum_err -= 0 - return sum_err / m - -def save_model(file_name, w): - '''保存最终的模型 - input: file_name(string):模型保存的文件名 - w(mat):LR模型的权重 - ''' - m = np.shape(w)[0] - f_w = open(file_name, "w") - w_array = [] - for i in xrange(m): - w_array.append(str(w[i, 0])) - f_w.write("\t".join(w_array)) - f_w.close() - -if __name__ == "__main__": - # 1、导入训练数据 - print "---------- 1.load data ------------" - feature, label = load_data("data.txt") - # 2、训练LR模型 - print "---------- 2.training ------------" - w = lr_train_bgd(feature, label, 1000, 0.01) - # 3、保存最终的模型 - print "---------- 3.save model ------------" - save_model("weights", w) - +# coding:UTF-8 +''' +Date:20160901 +@author: zhaozhiyong +''' +import numpy as np + +def load_data(file_name): + '''导入训练数据 + input: file_name(string)训练数据的位置 + output: feature_data(mat)特征 + label_data(mat)标签 + ''' + f = open(file_name) # 打开文件 + feature_data = [] + label_data = [] + for line in f.readlines(): + feature_tmp = [] + lable_tmp = [] + lines = line.strip().split("\t") + feature_tmp.append(1) # 偏置项 + for i in range(len(lines) - 1): + feature_tmp.append(float(lines[i])) + lable_tmp.append(float(lines[-1])) + + feature_data.append(feature_tmp) + label_data.append(lable_tmp) + f.close() # 关闭文件 + return np.mat(feature_data), np.mat(label_data) + +def sig(x): + '''Sigmoid函数 + input: x(mat):feature * w + output: sigmoid(x)(mat):Sigmoid值 + ''' + return 1.0 / (1 + np.exp(-x)) + +def lr_train_bgd(feature, label, maxCycle, alpha): + '''利用梯度下降法训练LR模型 + input: feature(mat)特征 + label(mat)标签 + maxCycle(int)最大迭代次数 + alpha(float)学习率 + output: w(mat):权重 + ''' + n = np.shape(feature)[1] # 特征个数 + w = np.mat(np.ones((n, 1))) # 初始化权重 + i = 0 + while i <= maxCycle: # 在最大迭代次数的范围内 + i += 1 # 当前的迭代次数 + h = sig(feature * w) # 计算Sigmoid值 + err = label - h + if i % 100 == 0: + print ("\t---------iter=" + str(i) + \ + " , train error rate= " + str(error_rate(h, label))) + w = w + alpha * feature.T * err # 权重修正 + return w + +def error_rate(h, label): + '''计算当前的损失函数值 + input: h(mat):预测值 + label(mat):实际值 + output: err/m(float):错误率 + ''' + m = np.shape(h)[0] + + sum_err = 0.0 + for i in range(m): + if h[i, 0] > 0 and (1 - h[i, 0]) > 0: + sum_err -= (label[i,0] * np.log(h[i,0]) + \ + (1-label[i,0]) * np.log(1-h[i,0])) + else: + sum_err -= 0 + return sum_err / m + +def save_model(file_name, w): + '''保存最终的模型 + input: file_name(string):模型保存的文件名 + w(mat):LR模型的权重 + ''' + m = np.shape(w)[0] + f_w = open(file_name, "w") + w_array = [] + for i in range(m): + w_array.append(str(w[i, 0])) + f_w.write("\t".join(w_array)) + f_w.close() + +if __name__ == "__main__": + # 1、导入训练数据 + print ("---------- 1.load data ------------") + feature, label = load_data("data.txt") + # 2、训练LR模型 + print ("---------- 2.training ------------") + w = lr_train_bgd(feature, label, 1000, 0.01) + # 3、保存最终的模型 + print ("---------- 3.save model ------------") + save_model("weights", w) + diff --git a/Chapter_4 SVM/svm.py b/Chapter_4 SVM/svm.py index 588eeb7..28d6d92 100644 --- a/Chapter_4 SVM/svm.py +++ b/Chapter_4 SVM/svm.py @@ -235,7 +235,7 @@ def svm_predict(svm, test_sample_x): kernel_value = cal_kernel_value(svm.train_x, test_sample_x, svm.kernel_opt) # 2、计算预测值 predict = kernel_value.T * np.multiply(svm.train_y, svm.alphas) + svm.b - return predict + return predict def cal_accuracy(svm, test_x, test_y): '''计算预测的准确性 diff --git a/Chapter_5 Random Forest/random_forests_train.py b/Chapter_5 Random Forest/random_forests_train.py index 1b987fb..057939f 100644 --- a/Chapter_5 Random Forest/random_forests_train.py +++ b/Chapter_5 Random Forest/random_forests_train.py @@ -1,155 +1,155 @@ -# coding:UTF-8 -''' -Date:20161030 -@author: zhaozhiyong -''' - -import numpy as np -import random as rd -from math import log -from tree import build_tree, predict -import cPickle as pickle - -def load_data(file_name): - '''导入数据 - input: file_name(string):训练数据保存的文件名 - output: data_train(list):训练数据 - ''' - data_train = [] - f = open(file_name) - for line in f.readlines(): - lines = line.strip().split("\t") - data_tmp = [] - for x in lines: - data_tmp.append(float(x)) - data_train.append(data_tmp) - f.close() - return data_train - -def choose_samples(data, k): - ''' - input: data(list):原始数据集 - k(int):选择特征的个数 - output: data_samples(list):被选择出来的样本 - feature(list):被选择的特征index - ''' - m, n = np.shape(data) # 样本的个数和样本特征的个数 - # 1、选择出k个特征的index - feature = [] - for j in xrange(k): - feature.append(rd.randint(0, n - 2)) # n-1列是标签 - # 2、选择出m个样本的index - index = [] - for i in xrange(m): - index.append(rd.randint(0, m - 1)) - # 3、从data中选择出m个样本的k个特征，组成数据集data_samples - data_samples = [] - for i in xrange(m): - data_tmp = [] - for fea in feature: - data_tmp.append(data[index[i]][fea]) - data_tmp.append(data[index[i]][-1]) - data_samples.append(data_tmp) - return data_samples, feature - - -def random_forest_training(data_train, trees_num): - '''构建随机森林 - input: data_train(list):训练数据 - trees_num(int):分类树的个数 - output: trees_result(list):每一棵树的最好划分 - trees_feature(list):每一棵树中对原始特征的选择 - ''' - trees_result = [] # 构建好每一棵树的最好划分 - trees_feature = [] - n = np.shape(data_train)[1] # 样本的维数 - if n > 2: - k = int(log(n - 1, 2)) + 1 # 设置特征的个数 - else: - k = 1 - # 开始构建每一棵树 - for i in xrange(trees_num): - # 1、随机选择m个样本, k个特征 - data_samples, feature = choose_samples(data_train, k) - # 2、构建每一棵分类树 - tree = build_tree(data_samples) - # 3、保存训练好的分类树 - trees_result.append(tree) - # 4、保存好该分类树使用到的特征 - trees_feature.append(feature) - - return trees_result, trees_feature - -def split_data(data_train, feature): - '''选择特征 - input: data_train(list):训练数据集 - feature(list):要选择的特征 - output: data(list):选择出来的数据集 - ''' - m = np.shape(data_train)[0] - data = [] - - for i in xrange(m): - data_x_tmp = [] - for x in feature: - data_x_tmp.append(data_train[i][x]) - data_x_tmp.append(data_train[i][-1]) - data.append(data_x_tmp) - return data - - -def get_predict(trees_result, trees_fiture, data_train): - m_tree = len(trees_result) - m = np.shape(data_train)[0] - - result = [] - for i in xrange(m_tree): - clf = trees_result[i] - feature = trees_fiture[i] - data = split_data(data_train, feature) - result_i = [] - for i in xrange(m): - result_i.append((predict(data[i][0:-1], clf).keys())[0]) - result.append(result_i) - final_predict = np.sum(result, axis=0) - return final_predict - -def cal_correct_rate(data_train, final_predict): - m = len(final_predict) - corr = 0.0 - for i in xrange(m): - if data_train[i][-1] * final_predict[i] > 0: - corr += 1 - return corr / m - -def save_model(trees_result, trees_feature, result_file, feature_file): - # 1、保存选择的特征 - m = len(trees_feature) - f_fea = open(feature_file, "w") - for i in xrange(m): - fea_tmp = [] - for x in trees_feature[i]: - fea_tmp.append(str(x)) - f_fea.writelines("\t".join(fea_tmp) + "\n") - f_fea.close() - - # 2、保存最终的随机森林模型 - with open(result_file, 'w') as f: - pickle.dump(trees_result, f) - - -if __name__ == "__main__": - # 1、导入数据 - print "----------- 1、load data -----------" - data_train = load_data("data.txt") - # 2、训练random_forest模型 - print "----------- 2、random forest training ------------" - trees_result, trees_feature = random_forest_training(data_train, 50) - # 3、得到训练的准确性 - print "------------ 3、get prediction correct rate ------------" - result = get_predict(trees_result, trees_feature, data_train) - corr_rate = cal_correct_rate(data_train, result) - print "\t------correct rate: ", corr_rate - # 4、保存最终的随机森林模型 - print "------------ 4、save model -------------" - save_model(trees_result, trees_feature, "result_file", "feature_file") +# coding:UTF-8 +''' +Date:20161030 +@author: zhaozhiyong +''' + +import numpy as np +import random as rd +from math import log +from tree import build_tree, predict +import pickle as pickle + +def load_data(file_name): + '''导入数据 + input: file_name(string):训练数据保存的文件名 + output: data_train(list):训练数据 + ''' + data_train = [] + f = open(file_name) + for line in f.readlines(): + lines = line.strip().split("\t") + data_tmp = [] + for x in lines: + data_tmp.append(float(x)) #因为行内做了拆分，所以先把一行的数据放到data_tmp中 + data_train.append(data_tmp) + f.close() + return data_train + +def choose_samples(data, k): + ''' + input: data(list):原始数据集 + k(int):选择特征的个数 + output: data_samples(list):被选择出来的样本 + feature(list):被选择的特征index + ''' + m, n = np.shape(data) # 样本的个数和样本特征的个数 + # 1、选择出k个特征的index + feature = [] + for j in range(k): + feature.append(rd.randint(0, n - 2)) # n-1列是标签 生成在半开半闭区间[low,high)上离散均匀分布的整数值;若high=None，则取值区间变为[0,low) + # 2、选择出m个样本的index + index = [] + for i in range(m): + index.append(rd.randint(0, m - 1)) + # 3、从data中选择出m个样本的k个特征，组成数据集data_samples + data_samples = [] + for i in range(m): + data_tmp = [] + for fea in feature: + data_tmp.append(data[index[i]][fea]) + data_tmp.append(data[index[i]][-1]) + data_samples.append(data_tmp) + return data_samples, feature + + +def random_forest_training(data_train, trees_num): + '''构建随机森林 + input: data_train(list):训练数据 + trees_num(int):分类树的个数 + output: trees_result(list):每一棵树的最好划分 + trees_feature(list):每一棵树中对原始特征的选择 + ''' + trees_result = [] # 构建好每一棵树的最好划分 + trees_feature = [] + n = np.shape(data_train)[1] # 样本的维数 + if n > 2: + k = int(log(n - 1, 2)) + 1 # 设置特征的个数 + else: + k = 1 + # 开始构建每一棵树 + for i in range(trees_num): + # 1、随机选择m个样本, k个特征 + data_samples, feature = choose_samples(data_train, k) + # 2、构建每一棵分类树 + tree = build_tree(data_samples) + # 3、保存训练好的分类树 + trees_result.append(tree) + # 4、保存好该分类树使用到的特征 + trees_feature.append(feature) + + return trees_result, trees_feature + +def split_data(data_train, feature): + '''选择特征 + input: data_train(list):训练数据集 + feature(list):要选择的特征 + output: data(list):选择出来的数据集 + ''' + m = np.shape(data_train)[0] + data = [] + + for i in range(m): + data_x_tmp = [] + for x in feature: + data_x_tmp.append(data_train[i][x]) + data_x_tmp.append(data_train[i][-1]) + data.append(data_x_tmp) + return data + + +def get_predict(trees_result, trees_fiture, data_train): + m_tree = len(trees_result) + m = np.shape(data_train)[0] + + result = [] + for i in range(m_tree): + clf = trees_result[i] + feature = trees_fiture[i] + data = split_data(data_train, feature) + result_i = [] + for i in range(m): + result_i.append(list((predict(data[i][0:-1], clf).keys()))[0]) # 这是由于python3改变了dict.keys,返回的是dict_keys对象,支持iterable 但不支持indexable，我们可以将其明确的转化成list + result.append(result_i) + final_predict = np.sum(result, axis=0) + return final_predict + +def cal_correct_rate(data_train, final_predict): + m = len(final_predict) + corr = 0.0 + for i in range(m): + if data_train[i][-1] * final_predict[i] > 0: + corr += 1 + return corr / m + +def save_model(trees_result, trees_feature, result_file, feature_file): + # 1、保存选择的特征 + m = len(trees_feature) + f_fea = open(feature_file, "w") + for i in range(m): + fea_tmp = [] + for x in trees_feature[i]: + fea_tmp.append(str(x)) + f_fea.writelines("\t".join(fea_tmp) + "\n") + f_fea.close() + + # 2、保存最终的随机森林模型 + with open(result_file, 'wb') as f: + pickle.dump(trees_result, f) + + +if __name__ == "__main__": + # 1、导入数据 + print ("----------- 1、load data -----------") + data_train = load_data("data.txt") + # 2、训练random_forest模型 + print ("----------- 2、random forest training ------------") + trees_result, trees_feature = random_forest_training(data_train, 50) + # 3、得到训练的准确性 + print ("------------ 3、get prediction correct rate ------------") + result = get_predict(trees_result, trees_feature, data_train) + corr_rate = cal_correct_rate(data_train, result) + print ("\t------correct rate: ", corr_rate) + # 4、保存最终的随机森林模型 + print ("------------ 4、save model -------------") + save_model(trees_result, trees_feature, "result_file", "feature_file") diff --git a/Chapter_5 Random Forest/tree.py b/Chapter_5 Random Forest/tree.py index cb6ead9..82146b2 100644 --- a/Chapter_5 Random Forest/tree.py +++ b/Chapter_5 Random Forest/tree.py @@ -1,131 +1,131 @@ -# coding:UTF-8 -''' -Date:20161030 -@author: zhaozhiyong -''' -from math import pow - -class node: - '''树的节点的类 - ''' - def __init__(self, fea=-1, value=None, results=None, right=None, left=None): - self.fea = fea # 用于切分数据集的属性的列索引值 - self.value = value # 设置划分的值 - self.results = results # 存储叶节点所属的类别 - self.right = right # 右子树 - self.left = left # 左子树 - -def split_tree(data, fea, value): - '''根据特征fea中的值value将数据集data划分成左右子树 - input: data(list):数据集 - fea(int):待分割特征的索引 - value(float):待分割的特征的具体值 - output: (set1,set2)(tuple):分割后的左右子树 - ''' - set_1 = [] - set_2 = [] - for x in data: - if x[fea] >= value: - set_1.append(x) - else: - set_2.append(x) - return (set_1, set_2) - -def label_uniq_cnt(data): - '''统计数据集中不同的类标签label的个数 - input: data(list):原始数据集 - output: label_uniq_cnt(int):样本中的标签的个数 - ''' - label_uniq_cnt = {} - - for x in data: - label = x[len(x) - 1] # 取得每一个样本的类标签label - if label not in label_uniq_cnt: - label_uniq_cnt[label] = 0 - label_uniq_cnt[label] = label_uniq_cnt[label] + 1 - return label_uniq_cnt - -def cal_gini_index(data): - '''计算给定数据集的Gini指数 - input: data(list):树中 - output: gini(float):Gini指数 - ''' - total_sample = len(data) # 样本的总个数 - if len(data) == 0: - return 0 - label_counts = label_uniq_cnt(data) # 统计数据集中不同标签的个数 - - # 计算数据集的Gini指数 - gini = 0 - for label in label_counts: - gini = gini + pow(label_counts[label], 2) - - gini = 1 - float(gini) / pow(total_sample, 2) - return gini - -def build_tree(data): - '''构建树 - input: data(list):训练样本 - output: node:树的根结点 - ''' - # 构建决策树，函数返回该决策树的根节点 - if len(data) == 0: - return node() - - # 1、计算当前的Gini指数 - currentGini = cal_gini_index(data) - - bestGain = 0.0 - bestCriteria = None # 存储最佳切分属性以及最佳切分点 - bestSets = None # 存储切分后的两个数据集 - - feature_num = len(data[0]) - 1 # 样本中特征的个数 - # 2、找到最好的划分 - for fea in range(0, feature_num): - # 2.1、取得fea特征处所有可能的取值 - feature_values = {} # 在fea位置处可能的取值 - for sample in data: # 对每一个样本 - feature_values[sample[fea]] = 1 # 存储特征fea处所有可能的取值 - - # 2.2、针对每一个可能的取值，尝试将数据集划分，并计算Gini指数 - for value in feature_values.keys(): # 遍历该属性的所有切分点 - # 2.2.1、 根据fea特征中的值value将数据集划分成左右子树 - (set_1, set_2) = split_tree(data, fea, value) - # 2.2.2、计算当前的Gini指数 - nowGini = float(len(set_1) * cal_gini_index(set_1) + \ - len(set_2) * cal_gini_index(set_2)) / len(data) - # 2.2.3、计算Gini指数的增加量 - gain = currentGini - nowGini - # 2.2.4、判断此划分是否比当前的划分更好 - if gain > bestGain and len(set_1) > 0 and len(set_2) > 0: - bestGain = gain - bestCriteria = (fea, value) - bestSets = (set_1, set_2) - - # 3、判断划分是否结束 - if bestGain > 0: - right = build_tree(bestSets[0]) - left = build_tree(bestSets[1]) - return node(fea=bestCriteria[0], value=bestCriteria[1], \ - right=right, left=left) - else: - return node(results=label_uniq_cnt(data)) # 返回当前的类别标签作为最终的类别标签 - -def predict(sample, tree): - '''对每一个样本sample进行预测 - input: sample(list):需要预测的样本 - tree(类):构建好的分类树 - output: tree.results:所属的类别 - ''' - # 1、只是树根 - if tree.results != None: - return tree.results - else: - # 2、有左右子树 - val_sample = sample[tree.fea] - branch = None - if val_sample >= tree.value: - branch = tree.right - else: - branch = tree.left - return predict(sample, branch) +# coding:UTF-8 +''' +Date:20161030 +@author: zhaozhiyong +''' +from math import pow + +class node: + '''树的节点的类 + ''' + def __init__(self, fea=-1, value=None, results=None, right=None, left=None): + self.fea = fea # 用于切分数据集的属性的列索引值 + self.value = value # 设置划分的值 + self.results = results # 存储叶节点所属的类别 + self.right = right # 右子树 + self.left = left # 左子树 + +def split_tree(data, fea, value): + '''根据特征fea中的值value将数据集data划分成左右子树 + input: data(list):数据集 + fea(int):待分割特征的索引 + value(float):待分割的特征的具体值 + output: (set1,set2)(tuple):分割后的左右子树 + ''' + set_1 = [] + set_2 = [] + for x in data: + if x[fea] >= value: + set_1.append(x) + else: + set_2.append(x) + return (set_1, set_2) + +def label_uniq_cnt(data): + '''统计数据集中不同的类标签label的个数 + input: data(list):原始数据集 + output: label_uniq_cnt(int):样本中的标签的个数 + ''' + label_uniq_cnt = {} + + for x in data: + label = x[len(x) - 1] # 取得每一个样本的类标签label + if label not in label_uniq_cnt: + label_uniq_cnt[label] = 0 #添加一个新key,初始化值是0 + label_uniq_cnt[label] = label_uniq_cnt[label] + 1 + return label_uniq_cnt + +def cal_gini_index(data): + '''计算给定数据集的Gini指数 + input: data(list):树中 + output: gini(float):Gini指数 + ''' + total_sample = len(data) # 样本的总个数 + if len(data) == 0: + return 0 + label_counts = label_uniq_cnt(data) # 统计数据集中不同标签的个数 + + # 计算数据集的Gini指数 + gini = 0 + for label in label_counts: + gini = gini + pow(label_counts[label], 2) + + gini = 1 - float(gini) / pow(total_sample, 2) + return gini + +def build_tree(data): + '''构建树 + input: data(list):训练样本 + output: node:树的根结点 + ''' + # 构建决策树，函数返回该决策树的根节点 + if len(data) == 0: + return node() + + # 1、计算当前的Gini指数 + currentGini = cal_gini_index(data) + + bestGain = 0.0 + bestCriteria = None # 存储最佳切分属性以及最佳切分点 + bestSets = None # 存储切分后的两个数据集 + + feature_num = len(data[0]) - 1 # 样本中特征的个数 (减去了标签列) + # 2、找到最好的划分 + for fea in range(0, feature_num): + # 2.1、取得fea特征处所有可能的取值 + feature_values = {} # 在fea位置处可能的取值 + for sample in data: # 对每一个样本 + feature_values[sample[fea]] = 1 # 存储特征fea处所有可能的取值 + + # 2.2、针对每一个可能的取值，尝试将数据集划分，并计算Gini指数 + for value in feature_values.keys(): # 遍历该属性的所有切分点 比如颜色：红绿蓝 三个切分点 + # 2.2.1、 根据fea特征中的值value将数据集划分成左右子树 + (set_1, set_2) = split_tree(data, fea, value) + # 2.2.2、计算当前的Gini指数 + nowGini = float(len(set_1) * cal_gini_index(set_1) + \ + len(set_2) * cal_gini_index(set_2)) / len(data) + # 2.2.3、计算Gini指数的增加量 + gain = currentGini - nowGini + # 2.2.4、判断此划分是否比当前的划分更好 + if gain > bestGain and len(set_1) > 0 and len(set_2) > 0: + bestGain = gain + bestCriteria = (fea, value) + bestSets = (set_1, set_2) + + # 3、判断划分是否结束 + if bestGain > 0: + right = build_tree(bestSets[0]) + left = build_tree(bestSets[1]) + return node(fea=bestCriteria[0], value=bestCriteria[1], \ + right=right, left=left) + else: + return node(results=label_uniq_cnt(data)) # 返回当前的类别标签作为最终的类别标签 + +def predict(sample, tree): + '''对每一个样本sample进行预测 + input: sample(list):需要预测的样本 + tree(类):构建好的分类树 + output: tree.results:所属的类别 + ''' + # 1、只是树根 + if tree.results != None: + return tree.results + else: + # 2、有左右子树 + val_sample = sample[tree.fea] + branch = None + if val_sample >= tree.value: + branch = tree.right + else: + branch = tree.left + return predict(sample, branch) diff --git a/Chapter_6 BP/bp_train.py b/Chapter_6 BP/bp_train.py index 3b4b4f7..eec44ee 100644 --- a/Chapter_6 BP/bp_train.py +++ b/Chapter_6 BP/bp_train.py @@ -68,9 +68,8 @@ def hidden_in(feature, w0, b0): hidden_in = feature * w0 for i in range(m): - #print(w0) - #print(hidden_in) - #print(hidden_in[i, ]) + #print(hidden_in[i, ],1) + #print(hidden_in[i,:],2) hidden_in[i, ] += b0 return hidden_in From 99defd96d3a83f0db040ee1e1fc94e15d2d6d9bf Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Thu, 29 Mar 2018 15:39:04 +0800 Subject: [PATCH 07/10] 1 --- .../softmax_regression_train.py | 193 +++++----- Chapter_3 Factorization Machine/FM_train.py | 362 +++++++++--------- Chapter_4 SVM/svm.py | 4 +- 3 files changed, 281 insertions(+), 278 deletions(-) diff --git a/Chapter_2 Softmax Regression/softmax_regression_train.py b/Chapter_2 Softmax Regression/softmax_regression_train.py index 583d7bf..b004af4 100644 --- a/Chapter_2 Softmax Regression/softmax_regression_train.py +++ b/Chapter_2 Softmax Regression/softmax_regression_train.py @@ -1,95 +1,98 @@ -# coding:UTF-8 -''' -Date:20160805 -@author: zhaozhiyong -''' -import numpy as np - -def load_data(inputfile): - '''导入训练数据 - input: inputfile(string)训练样本的位置 - output: feature_data(mat)特征 - label_data(mat)标签 - k(int)类别的个数 - ''' - f = open(inputfile) # 打开文件 - feature_data = [] - label_data = [] - for line in f.readlines(): - feature_tmp = [] - feature_tmp.append(1) # 偏置项 - lines = line.strip().split("\t") - for i in xrange(len(lines) - 1): - feature_tmp.append(float(lines[i])) - label_data.append(int(lines[-1])) - - feature_data.append(feature_tmp) - f.close() # 关闭文件 - return np.mat(feature_data), np.mat(label_data).T, len(set(label_data)) - -def cost(err, label_data): - '''计算损失函数值 - input: err(mat):exp的值 - label_data(mat):标签的值 - output: sum_cost / m(float):损失函数的值 - ''' - m = np.shape(err)[0] - sum_cost = 0.0 - for i in xrange(m): - if err[i, label_data[i, 0]] / np.sum(err[i, :]) > 0: - sum_cost -= np.log(err[i, label_data[i, 0]] / np.sum(err[i, :])) - else: - sum_cost -= 0 - return sum_cost / m - - -def gradientAscent(feature_data, label_data, k, maxCycle, alpha): - '''利用梯度下降法训练Softmax模型 - input: feature_data(mat):特征 - label_data(mat):标签 - k(int):类别的个数 - maxCycle(int):最大的迭代次数 - alpha(float):学习率 - output: weights(mat)：权重 - ''' - m, n = np.shape(feature_data) - weights = np.mat(np.ones((n, k))) # 权重的初始化 - i = 0 - while i <= maxCycle: - err = np.exp(feature_data * weights) - if i % 500 == 0: - print "\t-----iter: ", i , ", cost: ", cost(err, label_data) - rowsum = -err.sum(axis=1) - rowsum = rowsum.repeat(k, axis=1) - err = err / rowsum - for x in range(m): - err[x, label_data[x, 0]] += 1 - weights = weights + (alpha / m) * feature_data.T * err - i += 1 - return weights - -def save_model(file_name, weights): - '''保存最终的模型 - input: file_name(string):保存的文件名 - weights(mat):softmax模型 - ''' - f_w = open(file_name, "w") - m, n = np.shape(weights) - for i in xrange(m): - w_tmp = [] - for j in xrange(n): - w_tmp.append(str(weights[i, j])) - f_w.write("\t".join(w_tmp) + "\n") - f_w.close() - -if __name__ == "__main__": - inputfile = "SoftInput.txt" - # 1、导入训练数据 - print "---------- 1.load data ------------" - feature, label, k = load_data(inputfile) - # 2、训练Softmax模型 - print "---------- 2.training ------------" - weights = gradientAscent(feature, label, k, 10000, 0.4) - # 3、保存最终的模型 - print "---------- 3.save model ------------" - save_model("weights", weights) +# coding:UTF-8 +''' +Date:20160805 +@author: zhaozhiyong +''' +import numpy as np + +def load_data(inputfile): + '''导入训练数据 + input: inputfile(string)训练样本的位置 + output: feature_data(mat)特征 + label_data(mat)标签 + k(int)类别的个数 + ''' + f = open(inputfile) # 打开文件 + feature_data = [] + label_data = [] + for line in f.readlines(): + feature_tmp = [] + feature_tmp.append(1) # 偏置项 + lines = line.strip().split("\t") + for i in range(len(lines) - 1): + feature_tmp.append(float(lines[i])) + label_data.append(int(lines[-1])) + + feature_data.append(feature_tmp) + f.close() # 关闭文件 + return np.mat(feature_data), np.mat(label_data).T, len(set(label_data)) + +def cost(err, label_data): + '''计算损失函数值 + input: err(mat):exp的值 + label_data(mat):标签的值 + output: sum_cost / m(float):损失函数的值 + ''' + m = np.shape(err)[0] + sum_cost = 0.0 + for i in range(m): + if err[i, label_data[i, 0]] / np.sum(err[i, :]) > 0: + sum_cost -= np.log(err[i, label_data[i, 0]] / np.sum(err[i, :])) + else: + sum_cost -= 0 + return sum_cost / m + + +def gradientAscent(feature_data, label_data, k, maxCycle, alpha): + '''利用梯度下降法训练Softmax模型 + input: feature_data(mat):特征 + label_data(mat):标签 + k(int):类别的个数 + maxCycle(int):最大的迭代次数 + alpha(float):学习率 + output: weights(mat)：权重 + ''' + m, n = np.shape(feature_data) + weights = np.mat(np.ones((n, k))) # 权重的初始化 + i = 0 + while i <= maxCycle: + err = np.exp(feature_data * weights) #得到一个shape(m,k)的mat + + if i % 500 == 0: + print ("\t-----iter: ", i , ", cost: ", cost(err, label_data)) + rowsum = -err.sum(axis=1) #而当加入axis=1以后就是将一个矩阵的每一行向量相加 + rowsum = rowsum.repeat(k, axis=1) + # axis=0,沿着y轴复制，实际上增加了行数,axis=1,沿着x轴复制，实际上增加列数 + + err = err / rowsum + for x in range(m): + err[x, label_data[x, 0]] += 1 #得到的是标签的类型 + weights = weights + (alpha / m) * feature_data.T * err + i += 1 + return weights + +def save_model(file_name, weights): + '''保存最终的模型 + input: file_name(string):保存的文件名 + weights(mat):softmax模型 + ''' + f_w = open(file_name, "w") + m, n = np.shape(weights) + for i in range(m): + w_tmp = [] + for j in range(n): + w_tmp.append(str(weights[i, j])) + f_w.write("\t".join(w_tmp) + "\n") + f_w.close() + +if __name__ == "__main__": + inputfile = "SoftInput.txt" + # 1、导入训练数据 + print ("---------- 1.load data ------------") + feature, label, k = load_data(inputfile) + # 2、训练Softmax模型 + print ("---------- 2.training ------------") + weights = gradientAscent(feature, label, k, 10000, 0.4) + # 3、保存最终的模型 + print ("---------- 3.save model ------------") + save_model("weights", weights) diff --git a/Chapter_3 Factorization Machine/FM_train.py b/Chapter_3 Factorization Machine/FM_train.py index ba3d294..5fa6e56 100644 --- a/Chapter_3 Factorization Machine/FM_train.py +++ b/Chapter_3 Factorization Machine/FM_train.py @@ -1,181 +1,181 @@ -# coding:UTF-8 -''' -Date:20160831 -@author: zhaozhiyong -''' -import numpy as np -from random import normalvariate # 正态分布 - -def loadDataSet(data): - '''导入训练数据 - input: data(string)训练数据 - output: dataMat(list)特征 - labelMat(list)标签 - ''' - dataMat = [] - labelMat = [] - fr = open(data) # 打开文件 - for line in fr.readlines(): - lines = line.strip().split("\t") - lineArr = [] - - for i in xrange(len(lines) - 1): - lineArr.append(float(lines[i])) - dataMat.append(lineArr) - - labelMat.append(float(lines[-1]) * 2 - 1) # 转换成{-1,1} - fr.close() - return dataMat, labelMat - -def sigmoid(inx): - return 1.0 / (1 + np.exp(-inx)) - -def initialize_v(n, k): - '''初始化交叉项 - input: n(int)特征的个数 - k(int)FM模型的超参数 - output: v(mat):交叉项的系数权重 - ''' - v = np.mat(np.zeros((n, k))) - - for i in xrange(n): - for j in xrange(k): - # 利用正态分布生成每一个权重 - v[i, j] = normalvariate(0, 0.2) - return v - -def stocGradAscent(dataMatrix, classLabels, k, max_iter, alpha): - '''利用随机梯度下降法训练FM模型 - input: dataMatrix(mat)特征 - classLabels(mat)标签 - k(int)v的维数 - max_iter(int)最大迭代次数 - alpha(float)学习率 - output: w0(float),w(mat),v(mat):权重 - ''' - m, n = np.shape(dataMatrix) - # 1、初始化参数 - w = np.zeros((n, 1)) # 其中n是特征的个数 - w0 = 0 # 偏置项 - v = initialize_v(n, k) # 初始化V - - # 2、训练 - for it in xrange(max_iter): - for x in xrange(m): # 随机优化，对每一个样本而言的 - inter_1 = dataMatrix[x] * v - inter_2 = np.multiply(dataMatrix[x], dataMatrix[x]) * \ - np.multiply(v, v) # multiply对应元素相乘 - # 完成交叉项 - interaction = np.sum(np.multiply(inter_1, inter_1) - inter_2) / 2. - p = w0 + dataMatrix[x] * w + interaction # 计算预测的输出 - loss = sigmoid(classLabels[x] * p[0, 0]) - 1 - - w0 = w0 - alpha * loss * classLabels[x] - for i in xrange(n): - if dataMatrix[x, i] != 0: - w[i, 0] = w[i, 0] - alpha * loss * classLabels[x] * dataMatrix[x, i] - - for j in xrange(k): - v[i, j] = v[i, j] - alpha * loss * classLabels[x] * \ - (dataMatrix[x, i] * inter_1[0, j] -\ - v[i, j] * dataMatrix[x, i] * dataMatrix[x, i]) - - # 计算损失函数的值 - if it % 1000 == 0: - print "\t------- iter: ", it, " , cost: ", \ - getCost(getPrediction(np.mat(dataTrain), w0, w, v), classLabels) - - # 3、返回最终的FM模型的参数 - return w0, w, v - -def getCost(predict, classLabels): - '''计算预测准确性 - input: predict(list)预测值 - classLabels(list)标签 - output: error(float)计算损失函数的值 - ''' - m = len(predict) - error = 0.0 - for i in xrange(m): - error -= np.log(sigmoid(predict[i] * classLabels[i] )) - return error - -def getPrediction(dataMatrix, w0, w, v): - '''得到预测值 - input: dataMatrix(mat)特征 - w(int)常数项权重 - w0(int)一次项权重 - v(float)交叉项权重 - output: result(list)预测的结果 - ''' - m = np.shape(dataMatrix)[0] - result = [] - for x in xrange(m): - - inter_1 = dataMatrix[x] * v - inter_2 = np.multiply(dataMatrix[x], dataMatrix[x]) * \ - np.multiply(v, v) # multiply对应元素相乘 - # 完成交叉项 - interaction = np.sum(np.multiply(inter_1, inter_1) - inter_2) / 2. - p = w0 + dataMatrix[x] * w + interaction # 计算预测的输出 - pre = sigmoid(p[0, 0]) - result.append(pre) - return result - -def getAccuracy(predict, classLabels): - '''计算预测准确性 - input: predict(list)预测值 - classLabels(list)标签 - output: float(error) / allItem(float)错误率 - ''' - m = len(predict) - allItem = 0 - error = 0 - for i in xrange(m): - allItem += 1 - if float(predict[i]) < 0.5 and classLabels[i] == 1.0: - error += 1 - elif float(predict[i]) >= 0.5 and classLabels[i] == -1.0: - error += 1 - else: - continue - return float(error) / allItem - -def save_model(file_name, w0, w, v): - '''保存训练好的FM模型 - input: file_name(string):保存的文件名 - w0(float):偏置项 - w(mat):一次项的权重 - v(mat):交叉项的权重 - ''' - f = open(file_name, "w") - # 1、保存w0 - f.write(str(w0) + "\n") - # 2、保存一次项的权重 - w_array = [] - m = np.shape(w)[0] - for i in xrange(m): - w_array.append(str(w[i, 0])) - f.write("\t".join(w_array) + "\n") - # 3、保存交叉项的权重 - m1 , n1 = np.shape(v) - for i in xrange(m1): - v_tmp = [] - for j in xrange(n1): - v_tmp.append(str(v[i, j])) - f.write("\t".join(v_tmp) + "\n") - f.close() - - -if __name__ == "__main__": - # 1、导入训练数据 - print "---------- 1.load data ---------" - dataTrain, labelTrain = loadDataSet("data_1.txt") - print "---------- 2.learning ---------" - # 2、利用随机梯度训练FM模型 - w0, w, v = stocGradAscent(np.mat(dataTrain), labelTrain, 3, 10000, 0.01) - predict_result = getPrediction(np.mat(dataTrain), w0, w, v) # 得到训练的准确性 - print "----------training accuracy: %f" % (1 - getAccuracy(predict_result, labelTrain)) - print "---------- 3.save result ---------" - # 3、保存训练好的FM模型 - save_model("weights", w0, w, v) +# coding:UTF-8 +''' +Date:20160831 +@author: zhaozhiyong +''' +import numpy as np +from random import normalvariate # 正态分布 + +def loadDataSet(data): + '''导入训练数据 + input: data(string)训练数据 + output: dataMat(list)特征 + labelMat(list)标签 + ''' + dataMat = [] + labelMat = [] + fr = open(data) # 打开文件 + for line in fr.readlines(): + lines = line.strip().split("\t") + lineArr = [] + + for i in range(len(lines) - 1): + lineArr.append(float(lines[i])) + dataMat.append(lineArr) + + labelMat.append(float(lines[-1]) * 2 - 1) # 转换成{-1,1} + fr.close() + return dataMat, labelMat + +def sigmoid(inx): + return 1.0 / (1 + np.exp(-inx)) + +def initialize_v(n, k): + '''初始化交叉项 + input: n(int)特征的个数 + k(int)FM模型的超参数 + output: v(mat):交叉项的系数权重 + ''' + v = np.mat(np.zeros((n, k))) + + for i in range(n): + for j in range(k): + # 利用正态分布生成每一个权重 + v[i, j] = normalvariate(0, 0.2) + return v + +def stocGradAscent(dataMatrix, classLabels, k, max_iter, alpha): + '''利用随机梯度下降法训练FM模型 + input: dataMatrix(mat)特征 + classLabels(mat)标签 + k(int)v的维数 + max_iter(int)最大迭代次数 + alpha(float)学习率 + output: w0(float),w(mat),v(mat):权重 + ''' + m, n = np.shape(dataMatrix) + # 1、初始化参数 + w = np.zeros((n, 1)) # 其中n是特征的个数 + w0 = 0 # 偏置项 + v = initialize_v(n, k) # 初始化V + + # 2、训练 + for it in range(max_iter): + for x in range(m): # 随机优化，对每一个样本而言的 + inter_1 = dataMatrix[x] * v #shape(1,k) + inter_2 = np.multiply(dataMatrix[x], dataMatrix[x]) * \ + np.multiply(v, v) # multiply对应元素相乘 + # 完成交叉项 + interaction = np.sum(np.multiply(inter_1, inter_1) - inter_2) / 2. + p = w0 + dataMatrix[x] * w + interaction # 计算预测的输出 + loss = sigmoid(classLabels[x] * p[0, 0]) - 1 + + w0 = w0 - alpha * loss * classLabels[x] + for i in range(n): + if dataMatrix[x, i] != 0: + w[i, 0] = w[i, 0] - alpha * loss * classLabels[x] * dataMatrix[x, i] + + for j in range(k): + v[i, j] = v[i, j] - alpha * loss * classLabels[x] * \ + (dataMatrix[x, i] * inter_1[0, j] -\ + v[i, j] * dataMatrix[x, i] * dataMatrix[x, i]) + + # 计算损失函数的值 + if it % 1000 == 0: + print ("\t------- iter: ", it, " , cost: ", \ + getCost(getPrediction(np.mat(dataTrain), w0, w, v), classLabels)) + + # 3、返回最终的FM模型的参数 + return w0, w, v + +def getCost(predict, classLabels): + '''计算预测准确性 + input: predict(list)预测值 + classLabels(list)标签 + output: error(float)计算损失函数的值 + ''' + m = len(predict) + error = 0.0 + for i in range(m): + error -= np.log(sigmoid(predict[i] * classLabels[i] )) + return error + +def getPrediction(dataMatrix, w0, w, v): + '''得到预测值 + input: dataMatrix(mat)特征 + w(int)常数项权重 + w0(int)一次项权重 + v(float)交叉项权重 + output: result(list)预测的结果 + ''' + m = np.shape(dataMatrix)[0] + result = [] + for x in range(m): + + inter_1 = dataMatrix[x] * v + inter_2 = np.multiply(dataMatrix[x], dataMatrix[x]) * \ + np.multiply(v, v) # multiply对应元素相乘 + # 完成交叉项 + interaction = np.sum(np.multiply(inter_1, inter_1) - inter_2) / 2. + p = w0 + dataMatrix[x] * w + interaction # 计算预测的输出 + pre = sigmoid(p[0, 0]) + result.append(pre) + return result + +def getAccuracy(predict, classLabels): + '''计算预测准确性 + input: predict(list)预测值 + classLabels(list)标签 + output: float(error) / allItem(float)错误率 + ''' + m = len(predict) + allItem = 0 + error = 0 + for i in range(m): + allItem += 1 + if float(predict[i]) < 0.5 and classLabels[i] == 1.0: + error += 1 + elif float(predict[i]) >= 0.5 and classLabels[i] == -1.0: + error += 1 + else: + continue + return float(error) / allItem + +def save_model(file_name, w0, w, v): + '''保存训练好的FM模型 + input: file_name(string):保存的文件名 + w0(float):偏置项 + w(mat):一次项的权重 + v(mat):交叉项的权重 + ''' + f = open(file_name, "w") + # 1、保存w0 + f.write(str(w0) + "\n") + # 2、保存一次项的权重 + w_array = [] + m = np.shape(w)[0] + for i in range(m): + w_array.append(str(w[i, 0])) + f.write("\t".join(w_array) + "\n") + # 3、保存交叉项的权重 + m1 , n1 = np.shape(v) + for i in range(m1): + v_tmp = [] + for j in range(n1): + v_tmp.append(str(v[i, j])) + f.write("\t".join(v_tmp) + "\n") + f.close() + + +if __name__ == "__main__": + # 1、导入训练数据 + print ("---------- 1.load data ---------") + dataTrain, labelTrain = loadDataSet("data_1.txt") + print ("---------- 2.learning ---------") + # 2、利用随机梯度训练FM模型 + w0, w, v = stocGradAscent(np.mat(dataTrain), labelTrain, 3, 10000, 0.01) + predict_result = getPrediction(np.mat(dataTrain), w0, w, v) # 得到训练的准确性 + print ("----------training accuracy: %f" % (1 - getAccuracy(predict_result, labelTrain))) + print ("---------- 3.save result ---------") + # 3、保存训练好的FM模型 + save_model("weights", w0, w, v) diff --git a/Chapter_4 SVM/svm.py b/Chapter_4 SVM/svm.py index 28d6d92..ddb9beb 100644 --- a/Chapter_4 SVM/svm.py +++ b/Chapter_4 SVM/svm.py @@ -250,7 +250,7 @@ def cal_accuracy(svm, test_x, test_y): # 对每一个样本得到预测值 predict=svm_predict(svm, test_x[i, :]) # 判断每一个样本的预测值与真实值是否一致 - if np.sign(predict) == np.sign(test_y[i]): + if np.sign(predict) == np.sign(test_y[i]): #The sign function returns -1 if x < 0, 0 if x==0, 1 if x > 0. correct += 1 accuracy = correct / n_samples return accuracy @@ -260,6 +260,6 @@ def save_svm_model(svm_model, model_file): input: svm_model:SVM模型 model_file(string):SVM模型需要保存到的文件 ''' - with open(model_file, 'w') as f: + with open(model_file, 'wb') as f: pickle.dump(svm_model, f) From fb3bd91b62344b8c3f597557ffc86523df186716 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 7 May 2018 16:08:56 +0800 Subject: [PATCH 08/10] 11 --- Chapter_3 Factorization Machine/FM_train.py | 2 +- Chapter_4 SVM/svm.py | 2 +- Chapter_4 SVM/svm_train.py | 110 ++++++------ .../random_forests_test.py | 168 +++++++++--------- 4 files changed, 141 insertions(+), 141 deletions(-) diff --git a/Chapter_3 Factorization Machine/FM_train.py b/Chapter_3 Factorization Machine/FM_train.py index 5fa6e56..b1422f4 100644 --- a/Chapter_3 Factorization Machine/FM_train.py +++ b/Chapter_3 Factorization Machine/FM_train.py @@ -73,7 +73,7 @@ def stocGradAscent(dataMatrix, classLabels, k, max_iter, alpha): w0 = w0 - alpha * loss * classLabels[x] for i in range(n): if dataMatrix[x, i] != 0: - w[i, 0] = w[i, 0] - alpha * loss * classLabels[x] * dataMatrix[x, i] + w[i, 0] = w[i, 0] - alpha * loss * classLabels[x] * dataMatrix[x, i] for j in range(k): v[i, j] = v[i, j] - alpha * loss * classLabels[x] * \ diff --git a/Chapter_4 SVM/svm.py b/Chapter_4 SVM/svm.py index ddb9beb..87c7612 100644 --- a/Chapter_4 SVM/svm.py +++ b/Chapter_4 SVM/svm.py @@ -199,7 +199,7 @@ def SVM_training(train_x, train_y, C, toler, max_iter, kernel_option = ('rbf', 0 iteration = 0 while (iteration < max_iter) and ((alpha_pairs_changed > 0) or entireSet): - print "\t iterration: ", iteration + print ("\t iterration: ", iteration) alpha_pairs_changed = 0 if entireSet: diff --git a/Chapter_4 SVM/svm_train.py b/Chapter_4 SVM/svm_train.py index 46f5312..d199396 100644 --- a/Chapter_4 SVM/svm_train.py +++ b/Chapter_4 SVM/svm_train.py @@ -1,55 +1,55 @@ -# coding:UTF-8 - -import numpy as np -import svm - -def load_data_libsvm(data_file): - '''导入训练数据 - input: data_file(string):训练数据所在文件 - output: data(mat):训练样本的特征 - label(mat):训练样本的标签 - ''' - data = [] - label = [] - f = open(data_file) - for line in f.readlines(): - lines = line.strip().split(' ') - - # 提取得出label - label.append(float(lines[0])) - # 提取出特征，并将其放入到矩阵中 - index = 0 - tmp = [] - for i in xrange(1, len(lines)): - li = lines[i].strip().split(":") - if int(li[0]) - 1 == index: - tmp.append(float(li[1])) - else: - while(int(li[0]) - 1 > index): - tmp.append(0) - index += 1 - tmp.append(float(li[1])) - index += 1 - while len(tmp) < 13: - tmp.append(0) - data.append(tmp) - f.close() - return np.mat(data), np.mat(label).T - -if __name__ == "__main__": - # 1、导入训练数据 - print "------------ 1、load data --------------" - dataSet, labels = load_data_libsvm("heart_scale") - # 2、训练SVM模型 - print "------------ 2、training ---------------" - C = 0.6 - toler = 0.001 - maxIter = 500 - svm_model = svm.SVM_training(dataSet, labels, C, toler, maxIter) - # 3、计算训练的准确性 - print "------------ 3、cal accuracy --------------" - accuracy = svm.cal_accuracy(svm_model, dataSet, labels) - print "The training accuracy is: %.3f%%" % (accuracy * 100) - # 4、保存最终的SVM模型 - print "------------ 4、save model ----------------" - svm.save_svm_model(svm_model, "model_file") +# coding:UTF-8 + +import numpy as np +import svm + +def load_data_libsvm(data_file): + '''导入训练数据 + input: data_file(string):训练数据所在文件 + output: data(mat):训练样本的特征 + label(mat):训练样本的标签 + ''' + data = [] + label = [] + f = open(data_file) + for line in f.readlines(): + lines = line.strip().split(' ') + + # 提取得出label + label.append(float(lines[0])) + # 提取出特征，并将其放入到矩阵中 + index = 0 + tmp = [] + for i in range(1, len(lines)): + li = lines[i].strip().split(":") + if int(li[0]) - 1 == index: + tmp.append(float(li[1])) + else: + while(int(li[0]) - 1 > index): #列缺少的情况，先补0再取值 + tmp.append(0) + index += 1 + tmp.append(float(li[1])) + index += 1 + while len(tmp) < 13: + tmp.append(0) + data.append(tmp) + f.close() + return np.mat(data), np.mat(label).T + +if __name__ == "__main__": + # 1、导入训练数据 + print ("------------ 1、load data --------------") + dataSet, labels = load_data_libsvm("heart_scale") + # 2、训练SVM模型 + print ("------------ 2、training ---------------") + C = 0.6 + toler = 0.001 + maxIter = 500 + svm_model = svm.SVM_training(dataSet, labels, C, toler, maxIter) + # 3、计算训练的准确性 + print ("------------ 3、cal accuracy --------------") + accuracy = svm.cal_accuracy(svm_model, dataSet, labels) + print ("The training accuracy is: %.3f%%" % (accuracy * 100)) + # 4、保存最终的SVM模型 + print ("------------ 4、save model ----------------") + svm.save_svm_model(svm_model, "model_file") diff --git a/Chapter_5 Random Forest/random_forests_test.py b/Chapter_5 Random Forest/random_forests_test.py index d6387a2..8459fe1 100644 --- a/Chapter_5 Random Forest/random_forests_test.py +++ b/Chapter_5 Random Forest/random_forests_test.py @@ -1,84 +1,84 @@ -#coding:UTF-8 -''' -Date:20161030 -@author: zhaozhiyong -''' - -import cPickle as pickle -from random_forests_train import get_predict - -def load_data(file_name): - '''导入待分类的数据集 - input: file_name(string):待分类数据存储的位置 - output: test_data(list) - ''' - f = open(file_name) - test_data = [] - for line in f.readlines(): - lines = line.strip().split("\t") - tmp = [] - for x in lines: - tmp.append(float(x)) - tmp.append(0) # 保存初始的label - test_data.append(tmp) - f.close() - return test_data - -def load_model(result_file, feature_file): - '''导入随机森林模型和每一个分类树中选择的特征 - input: result_file(string):随机森林模型存储的文件 - feature_file(string):分类树选择的特征存储的文件 - output: trees_result(list):随机森林模型 - trees_fiture(list):每一棵分类树选择的特征 - ''' - # 1、导入选择的特征 - trees_fiture = [] - f_fea = open(feature_file) - for line in f_fea.readlines(): - lines = line.strip().split("\t") - tmp = [] - for x in lines: - tmp.append(int(x)) - trees_fiture.append(tmp) - f_fea.close() - - # 2、导入随机森林模型 - with open(result_file, 'r') as f: - trees_result = pickle.load(f) - - return trees_result, trees_fiture - -def save_result(data_test, prediction, result_file): - '''保存最终的预测结果 - input: data_test(list):待预测的数据 - prediction(list):预测的结果 - result_file(string):存储最终预测结果的文件名 - ''' - m = len(prediction) - n = len(data_test[0]) - - f_result = open(result_file, "w") - for i in xrange(m): - tmp = [] - for j in xrange(n -1): - tmp.append(str(data_test[i][j])) - tmp.append(str(prediction[i])) - f_result.writelines("\t".join(tmp) + "\n") - f_result.close() - -if __name__ == "__main__": - # 1、导入测试数据集 - print "--------- 1、load test data --------" - data_test = load_data("test_data.txt") - # 2、导入随机森林模型 - print "--------- 2、load random forest model ----------" - trees_result, trees_feature = load_model("result_file", "feature_file") - # 3、预测 - print "--------- 3、get prediction -----------" - prediction = get_predict(trees_result, trees_feature, data_test) - # 4、保存最终的预测结果 - print "--------- 4、save result -----------" - save_result(data_test, prediction, "final_result") - - - +#coding:UTF-8 +''' +Date:20161030 +@author: zhaozhiyong +''' + +import pickle as pickle +from random_forests_train import get_predict + +def load_data(file_name): + '''导入待分类的数据集 + input: file_name(string):待分类数据存储的位置 + output: test_data(list) + ''' + f = open(file_name) + test_data = [] + for line in f.readlines(): + lines = line.strip().split("\t") + tmp = [] + for x in lines: + tmp.append(float(x)) + tmp.append(0) # 保存初始的label + test_data.append(tmp) + f.close() + return test_data + +def load_model(result_file, feature_file): + '''导入随机森林模型和每一个分类树中选择的特征 + input: result_file(string):随机森林模型存储的文件 + feature_file(string):分类树选择的特征存储的文件 + output: trees_result(list):随机森林模型 + trees_fiture(list):每一棵分类树选择的特征 + ''' + # 1、导入选择的特征 + trees_fiture = [] + f_fea = open(feature_file) + for line in f_fea.readlines(): + lines = line.strip().split("\t") + tmp = [] + for x in lines: + tmp.append(int(x)) + trees_fiture.append(tmp) + f_fea.close() + + # 2、导入随机森林模型 + with open(result_file, 'r') as f: + trees_result = pickle.load(f) + + return trees_result, trees_fiture + +def save_result(data_test, prediction, result_file): + '''保存最终的预测结果 + input: data_test(list):待预测的数据 + prediction(list):预测的结果 + result_file(string):存储最终预测结果的文件名 + ''' + m = len(prediction) + n = len(data_test[0]) + + f_result = open(result_file, "w") + for i in xrange(m): + tmp = [] + for j in xrange(n -1): + tmp.append(str(data_test[i][j])) + tmp.append(str(prediction[i])) + f_result.writelines("\t".join(tmp) + "\n") + f_result.close() + +if __name__ == "__main__": + # 1、导入测试数据集 + print ("--------- 1、load test data --------") + data_test = load_data("test_data.txt") + # 2、导入随机森林模型 + print ("--------- 2、load random forest model ----------") + trees_result, trees_feature = load_model("result_file", "feature_file") + # 3、预测 + print ("--------- 3、get prediction -----------") + prediction = get_predict(trees_result, trees_feature, data_test) + # 4、保存最终的预测结果 + print ("--------- 4、save result -----------") + save_result(data_test, prediction, "final_result") + + + From 0a03810906714899de2be5a4e4d19185a294e8f3 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 11 Jun 2018 15:28:55 +0800 Subject: [PATCH 09/10] 111 --- Chapter12_DBSCAN/dbscan.py | 298 ++++++------ Chapter_11 MeanShift/mean_shift.py | 358 +++++++-------- Chapter_13 LabelPropagation/lb.py | 250 +++++----- .../item_based_recommend.py | 138 +++--- .../user_based_recommend.py | 232 +++++----- .../random_forests_test.py | 6 +- Chapter_5 Random Forest/tree.py | 2 +- .../linear_regression_train.py | 286 ++++++------ .../local_weight_regression.py | 62 +-- .../ridge_regression_train.py | 432 +++++++++--------- 10 files changed, 1040 insertions(+), 1024 deletions(-) diff --git a/Chapter12_DBSCAN/dbscan.py b/Chapter12_DBSCAN/dbscan.py index e846a47..57a5498 100644 --- a/Chapter12_DBSCAN/dbscan.py +++ b/Chapter12_DBSCAN/dbscan.py @@ -1,149 +1,149 @@ -# coding:UTF-8 -''' -Date:20160923 -@author: zhaozhiyong -''' - -import numpy as np -import math - -MinPts = 5 # 定义半径内的最少的数据点的个数 - -def load_data(file_path): - '''导入数据 - input: file_path(string):文件名 - output: data(mat):数据 - ''' - f = open(file_path) - data = [] - for line in f.readlines(): - data_tmp = [] - lines = line.strip().split("\t") - for x in lines: - data_tmp.append(float(x.strip())) - data.append(data_tmp) - f.close() - return np.mat(data) - -def epsilon(data, MinPts): - '''计算半径 - input: data(mat):训练数据 - MinPts(int):半径内的数据点的个数 - output: eps(float):半径 - ''' - m, n = np.shape(data) - xMax = np.max(data, 0) - xMin = np.min(data, 0) - eps = ((np.prod(xMax - xMin) * MinPts * math.gamma(0.5 * n + 1)) / (m * math.sqrt(math.pi ** n))) ** (1.0 / n) - return eps - -def distance(data): - m, n = np.shape(data) - dis = np.mat(np.zeros((m, m))) - for i in xrange(m): - for j in xrange(i, m): - # 计算i和j之间的欧式距离 - tmp = 0 - for k in xrange(n): - tmp += (data[i, k] - data[j, k]) * (data[i, k] - data[j, k]) - dis[i, j] = np.sqrt(tmp) - dis[j, i] = dis[i, j] - return dis - -def find_eps(distance_D, eps): - ind = [] - n = np.shape(distance_D)[1] - for j in xrange(n): - if distance_D[0, j] <= eps: - ind.append(j) - return ind - -def dbscan(data, eps, MinPts): - m = np.shape(data)[0] - # 区分核心点1，边界点0和噪音点-1 - types = np.mat(np.zeros((1, m))) - sub_class = np.mat(np.zeros((1, m))) - # 用于判断该点是否处理过，0表示未处理过 - dealed = np.mat(np.zeros((m, 1))) - # 计算每个数据点之间的距离 - dis = distance(data) - # 用于标记类别 - number = 1 - - # 对每一个点进行处理 - for i in xrange(m): - # 找到未处理的点 - if dealed[i, 0] == 0: - # 找到第i个点到其他所有点的距离 - D = dis[i, ] - # 找到半径eps内的所有点 - ind = find_eps(D, eps) - # 区分点的类型 - # 边界点 - if len(ind) > 1 and len(ind) < MinPts + 1: - types[0, i] = 0 - sub_class[0, i] = 0 - # 噪音点 - if len(ind) == 1: - types[0, i] = -1 - sub_class[0, i] = -1 - dealed[i, 0] = 1 - # 核心点 - if len(ind) >= MinPts + 1: - types[0, i] = 1 - for x in ind: - sub_class[0, x] = number - # 判断核心点是否密度可达 - while len(ind) > 0: - dealed[ind[0], 0] = 1 - D = dis[ind[0], ] - tmp = ind[0] - del ind[0] - ind_1 = find_eps(D, eps) - - if len(ind_1) > 1: # 处理非噪音点 - for x1 in ind_1: - sub_class[0, x1] = number - if len(ind_1) >= MinPts + 1: - types[0, tmp] = 1 - else: - types[0, tmp] = 0 - - for j in xrange(len(ind_1)): - if dealed[ind_1[j], 0] == 0: - dealed[ind_1[j], 0] = 1 - ind.append(ind_1[j]) - sub_class[0, ind_1[j]] = number - number += 1 - - # 最后处理所有未分类的点为噪音点 - ind_2 = ((sub_class == 0).nonzero())[1] - for x in ind_2: - sub_class[0, x] = -1 - types[0, x] = -1 - - return types, sub_class - -def save_result(file_name, source): - f = open(file_name, "w") - n = np.shape(source)[1] - tmp = [] - for i in xrange(n): - tmp.append(str(source[0, i])) - f.write("\n".join(tmp)) - f.close() - -if __name__ == "__main__": - # 1、导入数据 - print "----------- 1、load data ----------" - data = load_data("data.txt") - # 2、计算半径 - print "----------- 2、calculate eps ----------" - eps = epsilon(data, MinPts) - # 3、利用DBSCAN算法进行训练 - print "----------- 3、DBSCAN -----------" - types, sub_class = dbscan(data, eps, MinPts) - # 4、保存最终的结果 - print "----------- 4、save result -----------" - save_result("types", types) - save_result("sub_class", sub_class) +# coding:UTF-8 +''' +Date:20160923 +@author: zhaozhiyong +''' + +import numpy as np +import math + +MinPts = 5 # 定义半径内的最少的数据点的个数 + +def load_data(file_path): + '''导入数据 + input: file_path(string):文件名 + output: data(mat):数据 + ''' + f = open(file_path) + data = [] + for line in f.readlines(): + data_tmp = [] + lines = line.strip().split("\t") + for x in lines: + data_tmp.append(float(x.strip())) + data.append(data_tmp) + f.close() + return np.mat(data) + +def epsilon(data, MinPts): + '''计算半径 + input: data(mat):训练数据 + MinPts(int):半径内的数据点的个数 + output: eps(float):半径 + ''' + m, n = np.shape(data) + xMax = np.max(data, 0) + xMin = np.min(data, 0) + eps = ((np.prod(xMax - xMin) * MinPts * math.gamma(0.5 * n + 1)) / (m * math.sqrt(math.pi ** n))) ** (1.0 / n) + return eps + +def distance(data): + m, n = np.shape(data) + dis = np.mat(np.zeros((m, m))) + for i in range(m): + for j in range(i, m): + # 计算i和j之间的欧式距离 + tmp = 0 + for k in range(n): + tmp += (data[i, k] - data[j, k]) * (data[i, k] - data[j, k]) + dis[i, j] = np.sqrt(tmp) + dis[j, i] = dis[i, j] + return dis + +def find_eps(distance_D, eps): + ind = [] + n = np.shape(distance_D)[1] + for j in range(n): + if distance_D[0, j] <= eps: + ind.append(j) + return ind + +def dbscan(data, eps, MinPts): + m = np.shape(data)[0] + # 区分核心点1，边界点0和噪音点-1 + types = np.mat(np.zeros((1, m))) + sub_class = np.mat(np.zeros((1, m))) + # 用于判断该点是否处理过，0表示未处理过 + dealed = np.mat(np.zeros((m, 1))) + # 计算每个数据点之间的距离 + dis = distance(data) + # 用于标记类别 + number = 1 + + # 对每一个点进行处理 + for i in range(m): + # 找到未处理的点 + if dealed[i, 0] == 0: + # 找到第i个点到其他所有点的距离 + D = dis[i, ] + # 找到半径eps内的所有点 + ind = find_eps(D, eps) + # 区分点的类型 + # 边界点 + if len(ind) > 1 and len(ind) < MinPts + 1: + types[0, i] = 0 + sub_class[0, i] = 0 + # 噪音点 + if len(ind) == 1: + types[0, i] = -1 + sub_class[0, i] = -1 + dealed[i, 0] = 1 + # 核心点 + if len(ind) >= MinPts + 1: + types[0, i] = 1 + for x in ind: + sub_class[0, x] = number + # 判断核心点是否密度可达 + while len(ind) > 0: + dealed[ind[0], 0] = 1 + D = dis[ind[0], ] + tmp = ind[0] + del ind[0] #del删除的是变量，而不是数据。 + ind_1 = find_eps(D, eps) + + if len(ind_1) > 1: # 处理非噪音点 + for x1 in ind_1: + sub_class[0, x1] = number + if len(ind_1) >= MinPts + 1: + types[0, tmp] = 1 + else: + types[0, tmp] = 0 + + for j in range(len(ind_1)): + if dealed[ind_1[j], 0] == 0: + dealed[ind_1[j], 0] = 1 + ind.append(ind_1[j]) + sub_class[0, ind_1[j]] = number + number += 1 + + # 最后处理所有未分类的点为噪音点 + ind_2 = ((sub_class == 0).nonzero())[1] # nonzero():返回数组a中非零元素的索引值数组。 + for x in ind_2: + sub_class[0, x] = -1 + types[0, x] = -1 + + return types, sub_class + +def save_result(file_name, source): + f = open(file_name, "w") + n = np.shape(source)[1] + tmp = [] + for i in range(n): + tmp.append(str(source[0, i])) + f.write("\n".join(tmp)) + f.close() + +if __name__ == "__main__": + # 1、导入数据 + print ("----------- 1、load data ----------") + data = load_data("data.txt") + # 2、计算半径 + print ("----------- 2、calculate eps ----------") + eps = epsilon(data, MinPts) + # 3、利用DBSCAN算法进行训练 + print ("----------- 3、DBSCAN -----------") + types, sub_class = dbscan(data, eps, MinPts) + # 4、保存最终的结果 + print ("----------- 4、save result -----------") + save_result("types", types) + save_result("sub_class", sub_class) diff --git a/Chapter_11 MeanShift/mean_shift.py b/Chapter_11 MeanShift/mean_shift.py index 0f11b09..d98ddcb 100644 --- a/Chapter_11 MeanShift/mean_shift.py +++ b/Chapter_11 MeanShift/mean_shift.py @@ -1,179 +1,179 @@ -# coding:UTF-8 -''' -Date:20160426 -@author: zhaozhiyong -''' -import math -import numpy as np - -MIN_DISTANCE = 0.000001 # mini error - -def load_data(path, feature_num=2): - '''导入数据 - input: path(string)文件的存储位置 - feature_num(int)特征的个数 - output: data(array)特征 - ''' - f = open(path) # 打开文件 - data = [] - for line in f.readlines(): - lines = line.strip().split("\t") - data_tmp = [] - if len(lines) != feature_num: # 判断特征的个数是否正确 - continue - for i in xrange(feature_num): - data_tmp.append(float(lines[i])) - data.append(data_tmp) - f.close() # 关闭文件 - return data - -def gaussian_kernel(distance, bandwidth): - '''高斯核函数 - input: distance(mat):欧式距离 - bandwidth(int):核函数的带宽 - output: gaussian_val(mat):高斯函数值 - ''' - m = np.shape(distance)[0] # 样本个数 - right = np.mat(np.zeros((m, 1))) # mX1的矩阵 - for i in xrange(m): - right[i, 0] = (-0.5 * distance[i] * distance[i].T) / (bandwidth * bandwidth) - right[i, 0] = np.exp(right[i, 0]) - left = 1 / (bandwidth * math.sqrt(2 * math.pi)) - - gaussian_val = left * right - return gaussian_val - -def shift_point(point, points, kernel_bandwidth): - '''计算均值漂移点 - input: point(mat)需要计算的点 - points(array)所有的样本点 - kernel_bandwidth(int)核函数的带宽 - output: point_shifted(mat)漂移后的点 - ''' - points = np.mat(points) - m = np.shape(points)[0] # 样本的个数 - # 计算距离 - point_distances = np.mat(np.zeros((m, 1))) - for i in xrange(m): - point_distances[i, 0] = euclidean_dist(point, points[i]) - - # 计算高斯核 - point_weights = gaussian_kernel(point_distances, kernel_bandwidth) # mX1的矩阵 - - # 计算分母 - all_sum = 0.0 - for i in xrange(m): - all_sum += point_weights[i, 0] - - # 均值偏移 - point_shifted = point_weights.T * points / all_sum - return point_shifted - -def euclidean_dist(pointA, pointB): - '''计算欧式距离 - input: pointA(mat):A点的坐标 - pointB(mat):B点的坐标 - output: math.sqrt(total):两点之间的欧式距离 - ''' - # 计算pointA和pointB之间的欧式距离 - total = (pointA - pointB) * (pointA - pointB).T - return math.sqrt(total) # 欧式距离 - -def group_points(mean_shift_points): - '''计算所属的类别 - input: mean_shift_points(mat):漂移向量 - output: group_assignment(array):所属类别 - ''' - group_assignment = [] - m, n = np.shape(mean_shift_points) - index = 0 - index_dict = {} - for i in xrange(m): - item = [] - for j in xrange(n): - item.append(str(("%5.2f" % mean_shift_points[i, j]))) - - item_1 = "_".join(item) - if item_1 not in index_dict: - index_dict[item_1] = index - index += 1 - - for i in xrange(m): - item = [] - for j in xrange(n): - item.append(str(("%5.2f" % mean_shift_points[i, j]))) - - item_1 = "_".join(item) - group_assignment.append(index_dict[item_1]) - - return group_assignment - -def train_mean_shift(points, kenel_bandwidth=2): - '''训练Mean shift模型 - input: points(array):特征数据 - kenel_bandwidth(int):核函数的带宽 - output: points(mat):特征点 - mean_shift_points(mat):均值漂移点 - group(array):类别 - ''' - mean_shift_points = np.mat(points) - max_min_dist = 1 - iteration = 0 # 训练的代数 - m = np.shape(mean_shift_points)[0] # 样本的个数 - need_shift = [True] * m # 标记是否需要漂移 - - # 计算均值漂移向量 - while max_min_dist > MIN_DISTANCE: - max_min_dist = 0 - iteration += 1 - print "\titeration : " + str(iteration) - for i in range(0, m): - # 判断每一个样本点是否需要计算偏移均值 - if not need_shift[i]: - continue - p_new = mean_shift_points[i] - p_new_start = p_new - p_new = shift_point(p_new, points, kenel_bandwidth) # 对样本点进行漂移 - dist = euclidean_dist(p_new, p_new_start) # 计算该点与漂移后的点之间的距离 - - if dist > max_min_dist: - max_min_dist = dist - if dist < MIN_DISTANCE: # 不需要移动 - need_shift[i] = False - - mean_shift_points[i] = p_new - - # 计算最终的group - group = group_points(mean_shift_points) # 计算所属的类别 - - return np.mat(points), mean_shift_points, group - -def save_result(file_name, data): - '''保存最终的计算结果 - input: file_name(string):存储的文件名 - data(mat):需要保存的文件 - ''' - f = open(file_name, "w") - m, n = np.shape(data) - for i in xrange(m): - tmp = [] - for j in xrange(n): - tmp.append(str(data[i, j])) - f.write("\t".join(tmp) + "\n") - f.close() - - -if __name__ == "__main__": - # 导入数据集 - print "----------1.load data ------------" - data = load_data("data", 2) - # 训练，h=2 - print "----------2.training ------------" - points, shift_points, cluster = train_mean_shift(data, 2) - # 保存所属的类别文件 - print "----------3.1.save sub ------------" - save_result("sub_1", np.mat(cluster)) - print "----------3.2.save center ------------" - # 保存聚类中心 - save_result("center_1", shift_points) - +# coding:UTF-8 +''' +Date:20160426 +@author: zhaozhiyong +''' +import math +import numpy as np + +MIN_DISTANCE = 0.000001 # mini error + +def load_data(path, feature_num=2): + '''导入数据 + input: path(string)文件的存储位置 + feature_num(int)特征的个数 + output: data(array)特征 + ''' + f = open(path) # 打开文件 + data = [] + for line in f.readlines(): + lines = line.strip().split("\t") + data_tmp = [] + if len(lines) != feature_num: # 判断特征的个数是否正确 + continue + for i in range(feature_num): + data_tmp.append(float(lines[i])) + data.append(data_tmp) + f.close() # 关闭文件 + return data + +def gaussian_kernel(distance, bandwidth): + '''高斯核函数 + input: distance(mat):欧式距离 + bandwidth(int):核函数的带宽 + output: gaussian_val(mat):高斯函数值 + ''' + m = np.shape(distance)[0] # 样本个数 + right = np.mat(np.zeros((m, 1))) # mX1的矩阵 + for i in range(m): + right[i, 0] = (-0.5 * distance[i] * distance[i].T) / (bandwidth * bandwidth) + right[i, 0] = np.exp(right[i, 0]) + left = 1 / (bandwidth * math.sqrt(2 * math.pi)) + + gaussian_val = left * right + return gaussian_val + +def shift_point(point, points, kernel_bandwidth): + '''计算均值漂移点 + input: point(mat)需要计算的点 + points(array)所有的样本点 + kernel_bandwidth(int)核函数的带宽 + output: point_shifted(mat)漂移后的点 + ''' + points = np.mat(points) + m = np.shape(points)[0] # 样本的个数 + # 计算距离 + point_distances = np.mat(np.zeros((m, 1))) + for i in range(m): + point_distances[i, 0] = euclidean_dist(point, points[i]) + + # 计算高斯核 + point_weights = gaussian_kernel(point_distances, kernel_bandwidth) # mX1的矩阵 + + # 计算分母 + all_sum = 0.0 + for i in range(m): + all_sum += point_weights[i, 0] + + # 均值偏移 + point_shifted = point_weights.T * points / all_sum + return point_shifted + +def euclidean_dist(pointA, pointB): + '''计算欧式距离 + input: pointA(mat):A点的坐标 + pointB(mat):B点的坐标 + output: math.sqrt(total):两点之间的欧式距离 + ''' + # 计算pointA和pointB之间的欧式距离 + total = (pointA - pointB) * (pointA - pointB).T + return math.sqrt(total) # 欧式距离 + +def group_points(mean_shift_points): + '''计算所属的类别 + input: mean_shift_points(mat):漂移向量 + output: group_assignment(array):所属类别 + ''' + group_assignment = [] + m, n = np.shape(mean_shift_points) + index = 0 + index_dict = {} + for i in range(m): + item = [] + for j in range(n): + item.append(str(("%5.2f" % mean_shift_points[i, j]))) + + item_1 = "_".join(item) + if item_1 not in index_dict: + index_dict[item_1] = index + index += 1 + + for i in range(m): + item = [] + for j in range(n): + item.append(str(("%5.2f" % mean_shift_points[i, j]))) + + item_1 = "_".join(item) + group_assignment.append(index_dict[item_1]) + + return group_assignment + +def train_mean_shift(points, kenel_bandwidth=2): + '''训练Mean shift模型 + input: points(array):特征数据 + kenel_bandwidth(int):核函数的带宽 + output: points(mat):特征点 + mean_shift_points(mat):均值漂移点 + group(array):类别 + ''' + mean_shift_points = np.mat(points) + max_min_dist = 1 + iteration = 0 # 训练的代数 + m = np.shape(mean_shift_points)[0] #样本的个数 + need_shift = [True] * m # 标记是否需要漂移 + + # 计算均值漂移向量 + while max_min_dist > MIN_DISTANCE: + max_min_dist = 0 + iteration += 1 + print ("\titeration : " + str(iteration)) + for i in range(0, m): + # 判断每一个样本点是否需要计算偏移均值 + if not need_shift[i]: + continue + p_new = mean_shift_points[i] + p_new_start = p_new + p_new = shift_point(p_new, points, kenel_bandwidth) # 对样本点进行漂移 + dist = euclidean_dist(p_new, p_new_start) # 计算该点与漂移后的点之间的距离 + + if dist > max_min_dist: + max_min_dist = dist + if dist < MIN_DISTANCE: # 不需要移动 + need_shift[i] = False + + mean_shift_points[i] = p_new + + # 计算最终的group + group = group_points(mean_shift_points) # 计算所属的类别 + + return np.mat(points), mean_shift_points, group + +def save_result(file_name, data): + '''保存最终的计算结果 + input: file_name(string):存储的文件名 + data(mat):需要保存的文件 + ''' + f = open(file_name, "w") + m, n = np.shape(data) + for i in range(m): + tmp = [] + for j in range(n): + tmp.append(str(data[i, j])) + f.write("\t".join(tmp) + "\n") + f.close() + + +if __name__ == "__main__": + # 导入数据集 + print ("----------1.load data ------------") + data = load_data("data", 2) + # 训练，h=2 + print ("----------2.training ------------") + points, shift_points, cluster = train_mean_shift(data, 2) + # 保存所属的类别文件 + print ("----------3.1.save sub ------------") + save_result("sub_1", np.mat(cluster)) + print ("----------3.2.save center ------------") + # 保存聚类中心 + save_result("center_1", shift_points) + diff --git a/Chapter_13 LabelPropagation/lb.py b/Chapter_13 LabelPropagation/lb.py index 7eb286f..843688d 100644 --- a/Chapter_13 LabelPropagation/lb.py +++ b/Chapter_13 LabelPropagation/lb.py @@ -1,117 +1,133 @@ -# coding:UTF-8 -''' -Date:20160805 -@author: zhaozhiyong -''' -import string - -def loadData(filePath): - ''' - input: filePath(string)文件的存储位置 - output: vector_dict(dict)节点：社区 - edge_dict(dict)存储节点之间的边和权重 - ''' - f = open(filePath) - vector_dict = {} # 存储节点 - edge_dict = {} # 存储边 - for line in f.readlines(): - lines = line.strip().split("\t") - - for i in xrange(2): - if lines[i] not in vector_dict: # 节点已存储 - # 将节点放入到vector_dict中，设置所属社区为其自身 - vector_dict[lines[i]] = string.atoi(lines[i]) - # 将边放入到edge_dict - edge_list = [] - if len(lines) == 3: - edge_list.append(lines[1 - i] + ":" + lines[2]) - else: - edge_list.append(lines[1 - i] + ":" + "1") - edge_dict[lines[i]] = edge_list - else: # 节点未存储 - edge_list = edge_dict[lines[i]] - if len(lines) == 3: - edge_list.append(lines[1 - i] + ":" + lines[2]) - else: - edge_list.append(lines[1 - i] + ":" + "1") - edge_dict[lines[i]] = edge_list - f.close() - return vector_dict, edge_dict - -def get_max_community_label(vector_dict, adjacency_node_list): - '''得到相邻接的节点中标签数最多的标签 - input: vector_dict(dict)节点：社区 - adjacency_node_list(list)节点的邻接节点 - output: 节点所属的社区 - ''' - label_dict = {} - for node in adjacency_node_list: - node_id_weight = node.strip().split(":") - node_id = node_id_weight[0]#邻接节点 - node_weight = string.atoi(node_id_weight[1])#与邻接节点之间的权重 - if vector_dict[node_id] not in label_dict: - label_dict[vector_dict[node_id]] = node_weight - else: - label_dict[vector_dict[node_id]] += node_weight - - # 找到最大的标签 - sort_list = sorted(label_dict.items(), key=lambda d: d[1], reverse=True) - return sort_list[0][0] - -def check(vector_dict, edge_dict): - '''检查是否满足终止条件 - input: vector_dict(dict)节点：社区 - edge_dict(dict)存储节点之间的边和权重 - output: 是否需要更新 - ''' - for node in vector_dict.keys(): - adjacency_node_list = edge_dict[node] # 与节点node相连接的节点 - node_label = vector_dict[node] # 节点node所属社区 - label = get_max_community_label(vector_dict, adjacency_node_list) - if node_label == label: # 对每个节点，其所属的社区标签是最大的 - continue - else: - return 0 - return 1 - -def label_propagation(vector_dict, edge_dict): - '''标签传播 - input: vector_dict(dict)节点：社区 - edge_dict(dict)存储节点之间的边和权重 - output: vector_dict(dict)节点：社区 - ''' - # 初始化，设置每个节点属于不同的社区 - t = 0 - # 以随机的次序处理每个节点 - while True: - if (check(vector_dict, edge_dict) == 0): - t = t + 1 - print "iteration: ", t - # 对每一个node进行更新 - for node in vector_dict.keys(): - adjacency_node_list = edge_dict[node] # 获取节点node的邻接节点 - vector_dict[node] = get_max_community_label(vector_dict, adjacency_node_list) - print vector_dict - else: - break - return vector_dict - -def save_result(file_name, vec_new): - f_result = open(file_name, "w") - for key in vec_new.keys(): - f_result.write(str(key) + "\t" + str(vec_new[key]) + "\n") - f_result.close() - - -if __name__ == "__main__": - # 1、导入数据 - print "----------1.load data ------------" - vector_dict, edge_dict = loadData("cd_data.txt") - print "original community: \n", vector_dict - # 2、利用label propagation算法进行社区划分 - print "----------2.label propagation ------------" - vec_new = label_propagation(vector_dict, edge_dict) - # 3、保存最终的社区划分的结果 - print "----------3.save result ------------" - save_result("result1", vec_new) - print "final_result:", vec_new +# coding:UTF-8 +''' +Date:20160805 +@author: zhaozhiyong +''' +import string + +def loadData(filePath): + ''' + input: filePath(string)文件的存储位置 + output: vector_dict(dict)节点：社区 + edge_dict(dict)存储节点之间的边和权重 + ''' + f = open(filePath) + vector_dict = {} # 存储节点 + edge_dict = {} # 存储边 + for line in f.readlines(): + lines = line.strip().split("\t") + + for i in range(2):# 0 , 1 + if lines[i] not in vector_dict: # 节点已存储 + # 将节点放入到vector_dict中，设置所属社区为其自身 + vector_dict[lines[i]] = int(lines[i]) #字符串转换成浮点型 + # 将边放入到edge_dict + edge_list = [] + if len(lines) == 3: + edge_list.append(lines[1 - i] + ":" + lines[2]) + else: + edge_list.append(lines[1 - i] + ":" + "1") + edge_dict[lines[i]] = edge_list + else: # 节点未存储 + edge_list = edge_dict[lines[i]] + if len(lines) == 3: + edge_list.append(lines[1 - i] + ":" + lines[2]) + else: + edge_list.append(lines[1 - i] + ":" + "1") + edge_dict[lines[i]] = edge_list + f.close() + return vector_dict, edge_dict + +def get_max_community_label(vector_dict, adjacency_node_list): + '''得到相邻接的节点中标签数最多的标签 + input: vector_dict(dict)节点：社区 + adjacency_node_list(list)节点的邻接节点 + output: 节点所属的社区 + ''' + label_dict = {} + for node in adjacency_node_list: + node_id_weight = node.strip().split(":") + node_id = node_id_weight[0]#邻接节点 + node_weight = int(node_id_weight[1])#与邻接节点之间的权重 + if vector_dict[node_id] not in label_dict: + label_dict[vector_dict[node_id]] = node_weight + else: + label_dict[vector_dict[node_id]] += node_weight + + # 找到最大的标签 + sort_list = sorted(label_dict.items(), key=lambda d: d[1], reverse=True) #reverse = True进行降序排列、列表、数据项的方法 + return sort_list[0][0] + +def check(vector_dict, edge_dict): + '''检查是否满足终止条件 + input: vector_dict(dict)节点：社区 + edge_dict(dict)存储节点之间的边和权重 + output: 是否需要更新 + ''' + for node in vector_dict.keys(): + adjacency_node_list = edge_dict[node] # 与节点node相连接的节点 + node_label = vector_dict[node] # 节点node所属社区 + label = get_max_community_label(vector_dict, adjacency_node_list) + if node_label == label: # 对每个节点，其所属的社区标签是最大的 + continue + else: + return 0 + return 1 + +def label_propagation(vector_dict, edge_dict): + '''标签传播 + input: vector_dict(dict)节点：社区 + edge_dict(dict)存储节点之间的边和权重 + output: vector_dict(dict)节点：社区 + ''' + # 初始化，设置每个节点属于不同的社区 + t = 0 + # 以随机的次序处理每个节点 + while True: + if (check(vector_dict, edge_dict) == 0): + t = t + 1 + print ("iteration: ", t) + # 对每一个node进行更新 + for node in vector_dict.keys(): + adjacency_node_list = edge_dict[node] # 获取节点node的邻接节点 + vector_dict[node] = get_max_community_label(vector_dict, adjacency_node_list) + print (vector_dict) + else: + break + return vector_dict + +def save_result(file_name, vec_new): + f_result = open(file_name, "w") + for key in vec_new.keys(): + f_result.write(str(key) + "\t" + str(vec_new[key]) + "\n") + f_result.close() + + +if __name__ == "__main__": + # 1、导入数据 + print ("----------1.load data ------------") + vector_dict, edge_dict = loadData("cd_data.txt") + print ("original community: \n", vector_dict) + # 2、利用label propagation算法进行社区划分 + print ("----------2.label propagation ------------") + vec_new = label_propagation(vector_dict, edge_dict) + # 3、保存最终的社区划分的结果 + print ("----------3.save result ------------") + save_result("result1", vec_new) + print ("final_result:", vec_new) + + +''' + ----------1.load data ------------ +original community: + {'0': 0, '2': 2, '3': 3, '4': 4, '5': 5, '1': 1, '7': 7, '6': 6, '10': 10, '11': 11, '8': 8, '9': 9, '14': 14, '15': 15, '12': 12, '13': 13} +----------2.label propagation ------------ +iteration: 1 +{'0': 2, '2': 2, '3': 2, '4': 2, '5': 2, '1': 2, '7': 2, '6': 2, '10': 2, '11': 2, '8': 2, '9': 2, '14': 2, '15': 2, '12': 2, '13': 2} +----------3.save result ------------ +final_result: {'0': 2, '2': 2, '3': 2, '4': 2, '5': 2, '1': 2, '7': 2, '6': 2, '10': 2, '11': 2, '8': 2, '9': 2, '14': 2, '15': 2, '12': 2, '13': 2} + +''' + + + diff --git a/Chapter_14 CollaborativeFiltering/item_based_recommend.py b/Chapter_14 CollaborativeFiltering/item_based_recommend.py index 6fec4f5..d0758b0 100644 --- a/Chapter_14 CollaborativeFiltering/item_based_recommend.py +++ b/Chapter_14 CollaborativeFiltering/item_based_recommend.py @@ -1,69 +1,69 @@ -# coding:UTF-8 -''' -Date:20160928 -@author: zhaozhiyong -''' - -import numpy as np -from user_based_recommend import load_data, similarity - -def item_based_recommend(data, w, user): - '''基于商品相似度为用户user推荐商品 - input: data(mat):商品用户矩阵 - w(mat):商品与商品之间的相似性 - user(int):用户的编号 - output: predict(list):推荐列表 - ''' - m, n = np.shape(data) # m:商品数量 n:用户数量 - interaction = data[:,user].T # 用户user的互动商品信息 - - # 1、找到用户user没有互动的商品 - not_inter = [] - for i in xrange(n): - if interaction[0, i] == 0: # 用户user未打分项 - not_inter.append(i) - - # 2、对没有互动过的商品进行预测 - predict = {} - for x in not_inter: - item = np.copy(interaction) # 获取用户user对商品的互动信息 - for j in xrange(m): # 对每一个商品 - if item[0, j] != 0: # 利用互动过的商品预测 - if x not in predict: - predict[x] = w[x, j] * item[0, j] - else: - predict[x] = predict[x] + w[x, j] * item[0, j] - # 按照预测的大小从大到小排序 - return sorted(predict.items(), key=lambda d:d[1], reverse=True) - -def top_k(predict, k): - '''为用户推荐前k个商品 - input: predict(list):排好序的商品列表 - k(int):推荐的商品个数 - output: top_recom(list):top_k个商品 - ''' - top_recom = [] - len_result = len(predict) - if k >= len_result: - top_recom = predict - else: - for i in xrange(k): - top_recom.append(predict[i]) - return top_recom - -if __name__ == "__main__": - # 1、导入用户商品数据 - print "------------ 1. load data ------------" - data = load_data("data.txt") - # 将用户商品矩阵转置成商品用户矩阵 - data = data.T - # 2、计算商品之间的相似性 - print "------------ 2. calculate similarity between items -------------" - w = similarity(data) - # 3、利用用户之间的相似性进行预测评分 - print "------------ 3. predict ------------" - predict = item_based_recommend(data, w, 0) - # 4、进行Top-K推荐 - print "------------ 4. top_k recommendation ------------" - top_recom = top_k(predict, 2) - print top_recom +# coding:UTF-8 +''' +Date:20160928 +@author: zhaozhiyong +''' + +import numpy as np +from user_based_recommend import load_data, similarity + +def item_based_recommend(data, w, user): + '''基于商品相似度为用户user推荐商品 + input: data(mat):商品用户矩阵 + w(mat):商品与商品之间的相似性 + user(int):用户的编号 + output: predict(list):推荐列表 + ''' + m, n = np.shape(data) # m:商品数量 n:用户数量 + interaction = data[:,user].T # 用户user的互动商品信息 + + # 1、找到用户user没有互动的商品 + not_inter = [] + for i in range(n): + if interaction[0, i] == 0: # 用户user未打分项 + not_inter.append(i) + + # 2、对没有互动过的商品进行预测 + predict = {} + for x in not_inter: + item = np.copy(interaction) # 获取用户user对商品的互动信息 + for j in range(m): # 对每一个商品 + if item[0, j] != 0: # 利用互动过的商品预测 + if x not in predict: + predict[x] = w[x, j] * item[0, j] + else: + predict[x] = predict[x] + w[x, j] * item[0, j] + # 按照预测的大小从大到小排序 + return sorted(predict.items(), key=lambda d:d[1], reverse=True) + +def top_k(predict, k): + '''为用户推荐前k个商品 + input: predict(list):排好序的商品列表 + k(int):推荐的商品个数 + output: top_recom(list):top_k个商品 + ''' + top_recom = [] + len_result = len(predict) + if k >= len_result: + top_recom = predict + else: + for i in range(k): + top_recom.append(predict[i]) + return top_recom + +if __name__ == "__main__": + # 1、导入用户商品数据 + print ("------------ 1. load data ------------") + data = load_data("data.txt") + # 将用户商品矩阵转置成商品用户矩阵 + data = data.T + # 2、计算商品之间的相似性 + print ("------------ 2. calculate similarity between items -------------") + w = similarity(data) + # 3、利用用户之间的相似性进行预测评分 + print ("------------ 3. predict ------------" ) + predict = item_based_recommend(data, w, 0) + # 4、进行Top-K推荐 + print ("------------ 4. top_k recommendation ------------") + top_recom = top_k(predict, 2) + print top_recom diff --git a/Chapter_14 CollaborativeFiltering/user_based_recommend.py b/Chapter_14 CollaborativeFiltering/user_based_recommend.py index 072cd33..b02d2ee 100644 --- a/Chapter_14 CollaborativeFiltering/user_based_recommend.py +++ b/Chapter_14 CollaborativeFiltering/user_based_recommend.py @@ -1,116 +1,116 @@ -# coding:UTF-8 -''' -Date:20160928 -@author: zhaozhiyong -''' - -import numpy as np - -def load_data(file_path): - '''导入用户商品数据 - input: file_path(string):用户商品数据存放的文件 - output: data(mat):用户商品矩阵 - ''' - f = open(file_path) - data = [] - for line in f.readlines(): - lines = line.strip().split("\t") - tmp = [] - for x in lines: - if x != "-": - tmp.append(float(x)) # 直接存储用户对商品的打分 - else: - tmp.append(0) - data.append(tmp) - f.close() - - return np.mat(data) - -def cos_sim(x, y): - '''余弦相似性 - input: x(mat):以行向量的形式存储，可以是用户或者商品 - y(mat):以行向量的形式存储，可以是用户或者商品 - output: x和y之间的余弦相似度 - ''' - numerator = x * y.T # x和y之间的额内积 - denominator = np.sqrt(x * x.T) * np.sqrt(y * y.T) - return (numerator / denominator)[0, 0] - - -def similarity(data): - '''计算矩阵中任意两行之间的相似度 - input: data(mat):任意矩阵 - output: w(mat):任意两行之间的相似度 - ''' - m = np.shape(data)[0] # 用户的数量 - # 初始化相似度矩阵 - w = np.mat(np.zeros((m, m))) - - for i in xrange(m): - for j in xrange(i, m): - if j != i: - # 计算任意两行之间的相似度 - w[i, j] = cos_sim(data[i, ], data[j, ]) - w[j, i] = w[i, j] - else: - w[i, j] = 0 - return w - -def user_based_recommend(data, w, user): - '''基于用户相似性为用户user推荐商品 - input: data(mat):用户商品矩阵 - w(mat):用户之间的相似度 - user(int):用户的编号 - output: predict(list):推荐列表 - ''' - m, n = np.shape(data) - interaction = data[user, ] # 用户user与商品信息 - - # 1、找到用户user没有互动过的商品 - not_inter = [] - for i in xrange(n): - if interaction[0, i] == 0: # 没有互动的商品 - not_inter.append(i) - - # 2、对没有互动过的商品进行预测 - predict = {} - for x in not_inter: - item = np.copy(data[:, x]) # 找到所有用户对商品x的互动信息 - for i in xrange(m): # 对每一个用户 - if item[i, 0] != 0: # 若该用户对商品x有过互动 - if x not in predict: - predict[x] = w[user, i] * item[i, 0] - else: - predict[x] = predict[x] + w[user, i] * item[i, 0] - # 3、按照预测的大小从大到小排序 - return sorted(predict.items(), key=lambda d:d[1], reverse=True) - -def top_k(predict, k): - '''为用户推荐前k个商品 - input: predict(list):排好序的商品列表 - k(int):推荐的商品个数 - output: top_recom(list):top_k个商品 - ''' - top_recom = [] - len_result = len(predict) - if k >= len_result: - top_recom = predict - else: - for i in xrange(k): - top_recom.append(predict[i]) - return top_recom - -if __name__ == "__main__": - # 1、导入用户商品数据 - print "------------ 1. load data ------------" - data = load_data("data.txt") - # 2、计算用户之间的相似性 - print "------------ 2. calculate similarity between users -------------" - w = similarity(data) - # 3、利用用户之间的相似性进行推荐 - print "------------ 3. predict ------------" - predict = user_based_recommend(data, w, 0) - # 4、进行Top-K推荐 - print "------------ 4. top_k recommendation ------------" - top_recom = top_k(predict, 2) - print top_recom +# coding:UTF-8 +''' +Date:20160928 +@author: zhaozhiyong +''' + +import numpy as np + +def load_data(file_path): + '''导入用户商品数据 + input: file_path(string):用户商品数据存放的文件 + output: data(mat):用户商品矩阵 + ''' + f = open(file_path) + data = [] + for line in f.readlines(): + lines = line.strip().split("\t") + tmp = [] + for x in lines: + if x != "-": + tmp.append(float(x)) # 直接存储用户对商品的打分 + else: + tmp.append(0) + data.append(tmp) + f.close() + + return np.mat(data) + +def cos_sim(x, y): + '''余弦相似性 + input: x(mat):以行向量的形式存储，可以是用户或者商品 + y(mat):以行向量的形式存储，可以是用户或者商品 + output: x和y之间的余弦相似度 + ''' + numerator = x * y.T # x和y之间的额内积 + denominator = np.sqrt(x * x.T) * np.sqrt(y * y.T) + return (numerator / denominator)[0, 0] + + +def similarity(data): + '''计算矩阵中任意两行之间的相似度 + input: data(mat):任意矩阵 + output: w(mat):任意两行之间的相似度 + ''' + m = np.shape(data)[0] # 用户的数量 + # 初始化相似度矩阵 + w = np.mat(np.zeros((m, m))) + + for i in range(m): + for j in range(i, m): + if j != i: + # 计算任意两行之间的相似度 + w[i, j] = cos_sim(data[i, ], data[j, ]) + w[j, i] = w[i, j] + else: + w[i, j] = 0 + return w + +def user_based_recommend(data, w, user): + '''基于用户相似性为用户user推荐商品 + input: data(mat):用户商品矩阵 + w(mat):用户之间的相似度 + user(int):用户的编号 + output: predict(list):推荐列表 + ''' + m, n = np.shape(data) + interaction = data[user, ] # 用户user与商品信息 + + # 1、找到用户user没有互动过的商品 + not_inter = [] + for i in range(n): + if interaction[0, i] == 0: # 没有互动的商品 + not_inter.append(i) + + # 2、对没有互动过的商品进行预测 + predict = {} + for x in not_inter: + item = np.copy(data[:, x]) # 找到所有用户对商品x的互动信息 + for i in range(m): # 对每一个用户 + if item[i, 0] != 0: # 若该用户对商品x有过互动 + if x not in predict: + predict[x] = w[user, i] * item[i, 0] + else: + predict[x] = predict[x] + w[user, i] * item[i, 0] + # 3、按照预测的大小从大到小排序 + return sorted(predict.items(), key=lambda d:d[1], reverse=True) + +def top_k(predict, k): + '''为用户推荐前k个商品 + input: predict(list):排好序的商品列表 + k(int):推荐的商品个数 + output: top_recom(list):top_k个商品 + ''' + top_recom = [] + len_result = len(predict) + if k >= len_result: + top_recom = predict + else: + for i in range(k): + top_recom.append(predict[i]) + return top_recom + +if __name__ == "__main__": + # 1、导入用户商品数据 + print ("------------ 1. load data ------------") + data = load_data("data.txt") + # 2、计算用户之间的相似性 + print ("------------ 2. calculate similarity between users -------------") + w = similarity(data) + # 3、利用用户之间的相似性进行推荐 + print ("------------ 3. predict ------------") + predict = user_based_recommend(data, w, 0) + # 4、进行Top-K推荐 + print ("------------ 4. top_k recommendation ------------") + top_recom = top_k(predict, 2) + print top_recom diff --git a/Chapter_5 Random Forest/random_forests_test.py b/Chapter_5 Random Forest/random_forests_test.py index 8459fe1..abb26bf 100644 --- a/Chapter_5 Random Forest/random_forests_test.py +++ b/Chapter_5 Random Forest/random_forests_test.py @@ -43,7 +43,7 @@ def load_model(result_file, feature_file): f_fea.close() # 2、导入随机森林模型 - with open(result_file, 'r') as f: + with open(result_file, 'rb') as f: trees_result = pickle.load(f) return trees_result, trees_fiture @@ -58,9 +58,9 @@ def save_result(data_test, prediction, result_file): n = len(data_test[0]) f_result = open(result_file, "w") - for i in xrange(m): + for i in range(m): tmp = [] - for j in xrange(n -1): + for j in range(n -1): tmp.append(str(data_test[i][j])) tmp.append(str(prediction[i])) f_result.writelines("\t".join(tmp) + "\n") diff --git a/Chapter_5 Random Forest/tree.py b/Chapter_5 Random Forest/tree.py index 82146b2..20dbf2e 100644 --- a/Chapter_5 Random Forest/tree.py +++ b/Chapter_5 Random Forest/tree.py @@ -83,7 +83,7 @@ def build_tree(data): # 2、找到最好的划分 for fea in range(0, feature_num): # 2.1、取得fea特征处所有可能的取值 - feature_values = {} # 在fea位置处可能的取值 + feature_values = {} # 在fea位置处可能的取值 for sample in data: # 对每一个样本 feature_values[sample[fea]] = 1 # 存储特征fea处所有可能的取值 diff --git a/Chapter_7 LinearRegression/linear_regression_train.py b/Chapter_7 LinearRegression/linear_regression_train.py index 103eeec..ea21126 100644 --- a/Chapter_7 LinearRegression/linear_regression_train.py +++ b/Chapter_7 LinearRegression/linear_regression_train.py @@ -1,143 +1,143 @@ -# coding:UTF-8 - -import numpy as np -from math import pow - -def load_data(file_path): - '''导入数据 - input: file_path(string):训练数据 - output: feature(mat):特征 - label(mat):标签 - ''' - f = open(file_path) - feature = [] - label = [] - for line in f.readlines(): - feature_tmp = [] - lines = line.strip().split("\t") - feature_tmp.append(1) # x0 - for i in xrange(len(lines) - 1): - feature_tmp.append(float(lines[i])) - feature.append(feature_tmp) - label.append(float(lines[-1])) - f.close() - return np.mat(feature), np.mat(label).T - -def least_square(feature, label): - '''最小二乘法 - input: feature(mat):特征 - label(mat):标签 - output: w(mat):回归系数 - ''' - w = (feature.T * feature).I * feature.T * label - return w - -def first_derivativ(feature, label, w): - '''计算一阶导函数的值 - input: feature(mat):特征 - label(mat):标签 - output: g(mat):一阶导数值 - ''' - m, n = np.shape(feature) - g = np.mat(np.zeros((n, 1))) - for i in xrange(m): - err = label[i, 0] - feature[i, ] * w - for j in xrange(n): - g[j, ] -= err * feature[i, j] - return g - -def second_derivative(feature): - '''计算二阶导函数的值 - input: feature(mat):特征 - output: G(mat):二阶导数值 - ''' - m, n = np.shape(feature) - G = np.mat(np.zeros((n, n))) - for i in xrange(m): - x_left = feature[i, ].T - x_right = feature[i, ] - G += x_left * x_right - return G - -def get_error(feature, label, w): - '''计算误差 - input: feature(mat):特征 - label(mat):标签 - w(mat):线性回归模型的参数 - output: 损失函数值 - ''' - return (label - feature * w).T * (label - feature * w) / 2 - -def get_min_m(feature, label, sigma, delta, d, w, g): - '''计算步长中最小的值m - input: feature(mat):特征 - label(mat):标签 - sigma(float),delta(float):全局牛顿法的参数 - d(mat):负的一阶导数除以二阶导数值 - g(mat):一阶导数值 - output: m(int):最小m值 - ''' - m = 0 - while True: - w_new = w + pow(sigma, m) * d - left = get_error(feature, label , w_new) - right = get_error(feature, label , w) + delta * pow(sigma, m) * g.T * d - if left <= right: - break - else: - m += 1 - return m - -def newton(feature, label, iterMax, sigma, delta): - '''牛顿法 - input: feature(mat):特征 - label(mat):标签 - iterMax(int):最大迭代次数 - sigma(float), delta(float):牛顿法中的参数 - output: w(mat):回归系数 - ''' - n = np.shape(feature)[1] - w = np.mat(np.zeros((n, 1))) - it = 0 - while it <= iterMax: - # print it - g = first_derivativ(feature, label, w) # 一阶导数 - G = second_derivative(feature) # 二阶导数 - d = -G.I * g - m = get_min_m(feature, label, sigma, delta, d, w, g) # 得到最小的m - w = w + pow(sigma, m) * d - if it % 10 == 0: - print "\t---- itration: ", it, " , error: ", get_error(feature, label , w)[0, 0] - it += 1 - return w - -def save_model(file_name, w): - '''保存最终的模型 - input: file_name(string):要保存的文件的名称 - w(mat):训练好的线性回归模型 - ''' - f_result = open(file_name, "w") - m, n = np.shape(w) - for i in xrange(m): - w_tmp = [] - for j in xrange(n): - w_tmp.append(str(w[i, j])) - f_result.write("\t".join(w_tmp) + "\n") - f_result.close() - - -if __name__ == "__main__": - # 1、导入数据集 - print "----------- 1.load data ----------" - feature, label = load_data("data.txt") - # 2.1、最小二乘求解 - print "----------- 2.training ----------" - # print "\t ---------- least_square ----------" - # w_ls = least_square(feature, label) - # 2.2、牛顿法 - print "\t ---------- newton ----------" - w_newton = newton(feature, label, 50, 0.1, 0.5) - # 3、保存最终的结果 - print "----------- 3.save result ----------" - save_model("weights", w_newton) - +# coding:UTF-8 + +import numpy as np +from math import pow + +def load_data(file_path): + '''导入数据 + input: file_path(string):训练数据 + output: feature(mat):特征 + label(mat):标签 + ''' + f = open(file_path) + feature = [] + label = [] + for line in f.readlines(): + feature_tmp = [] + lines = line.strip().split("\t") + feature_tmp.append(1) # x0 + for i in range(len(lines) - 1): + feature_tmp.append(float(lines[i])) + feature.append(feature_tmp) + label.append(float(lines[-1])) + f.close() + return np.mat(feature), np.mat(label).T + +def least_square(feature, label): + '''最小二乘法 + input: feature(mat):特征 + label(mat):标签 + output: w(mat):回归系数 + ''' + w = (feature.T * feature).I * feature.T * label #.I 表示逆矩阵 + return w + +def first_derivativ(feature, label, w): + '''计算一阶导函数的值 + input: feature(mat):特征 + label(mat):标签 + output: g(mat):一阶导数值 + ''' + m, n = np.shape(feature) + g = np.mat(np.zeros((n, 1))) + for i in range(m): + err = label[i, 0] - feature[i, ] * w + for j in range(n): + g[j, ] -= err * feature[i, j] + return g + +def second_derivative(feature): + '''计算二阶导函数的值 + input: feature(mat):特征 + output: G(mat):二阶导数值 + ''' + m, n = np.shape(feature) + G = np.mat(np.zeros((n, n))) + for i in range(m): + x_left = feature[i, ].T + x_right = feature[i, ] + G += x_left * x_right + return G + +def get_error(feature, label, w): + '''计算误差 + input: feature(mat):特征 + label(mat):标签 + w(mat):线性回归模型的参数 + output: 损失函数值 + ''' + return (label - feature * w).T * (label - feature * w) / 2 + +def get_min_m(feature, label, sigma, delta, d, w, g): + '''计算步长中最小的值m + input: feature(mat):特征 + label(mat):标签 + sigma(float),delta(float):全局牛顿法的参数 + d(mat):负的一阶导数除以二阶导数值 + g(mat):一阶导数值 + output: m(int):最小m值 + ''' + m = 0 + while True: + w_new = w + pow(sigma, m) * d + left = get_error(feature, label , w_new) + right = get_error(feature, label , w) + delta * pow(sigma, m) * g.T * d + if left <= right: + break + else: + m += 1 + return m + +def newton(feature, label, iterMax, sigma, delta): + '''牛顿法 + input: feature(mat):特征 + label(mat):标签 + iterMax(int):最大迭代次数 + sigma(float), delta(float):牛顿法中的参数 + output: w(mat):回归系数 + ''' + n = np.shape(feature)[1] + w = np.mat(np.zeros((n, 1))) + it = 0 + while it <= iterMax: + # print it + g = first_derivativ(feature, label, w) # 一阶导数 + G = second_derivative(feature) # 二阶导数 + d = -G.I * g + m = get_min_m(feature, label, sigma, delta, d, w, g) # 得到最小的m + w = w + pow(sigma, m) * d + if it % 10 == 0: + print ("\t---- itration: ", it, " , error: ", get_error(feature, label , w)[0, 0]) + it += 1 + return w + +def save_model(file_name, w): + '''保存最终的模型 + input: file_name(string):要保存的文件的名称 + w(mat):训练好的线性回归模型 + ''' + f_result = open(file_name, "w") + m, n = np.shape(w) + for i in range(m): + w_tmp = [] + for j in range(n): + w_tmp.append(str(w[i, j])) + f_result.write("\t".join(w_tmp) + "\n") + f_result.close() + + +if __name__ == "__main__": + # 1、导入数据集 + print ("----------- 1.load data ----------") + feature, label = load_data("data.txt") + # 2.1、最小二乘求解 + print ("----------- 2.training ----------") + # print "\t ---------- least_square ----------" + # w_ls = least_square(feature, label) + # 2.2、牛顿法 + print ("\t ---------- newton ----------") + w_newton = newton(feature, label, 50, 0.1, 0.5) + # 3、保存最终的结果 + print ("----------- 3.save result ----------") + save_model("weights", w_newton) + diff --git a/Chapter_7 LinearRegression/local_weight_regression.py b/Chapter_7 LinearRegression/local_weight_regression.py index 24b32cd..73966a5 100644 --- a/Chapter_7 LinearRegression/local_weight_regression.py +++ b/Chapter_7 LinearRegression/local_weight_regression.py @@ -1,31 +1,31 @@ -# coding:UTF-8 - -import numpy as np -from linear_regression_train import load_data - -def lwlr(feature, label, k): - '''局部加权线性回归 - input: feature(mat):特征 - label(mat):标签 - k(int):核函数的系数 - output: predict(mat):最终的结果 - ''' - m = np.shape(feature)[0] - predict = np.zeros(m) - weights = np.mat(np.eye(m)) - for i in xrange(m): - for j in xrange(m): - diff = feature[i, ] - feature[j, ] - weights[j,j] = np.exp(diff * diff.T / (-2.0 * k ** 2)) - xTx = feature.T * (weights * feature) - ws = xTx.I * (feature.T * (weights * label)) - predict[i] = feature[i, ] * ws - return predict - -if __name__ == "__main__": - # 1、导入数据集 - feature, label = load_data("data.txt") - predict = lwlr(feature, label, 0.002) - m = np.shape(predict)[0] - for i in xrange(m): - print feature[i, 1], predict[i] +# coding:UTF-8 + +import numpy as np +from linear_regression_train import load_data + +def lwlr(feature, label, k): + '''局部加权线性回归 + input: feature(mat):特征 + label(mat):标签 + k(int):核函数的系数 + output: predict(mat):最终的结果 + ''' + m = np.shape(feature)[0] + predict = np.zeros(m) + weights = np.mat(np.eye(m)) + for i in range(m): + for j in range(m): + diff = feature[i, ] - feature[j, ] + weights[j,j] = np.exp(diff * diff.T / (-2.0 * k ** 2)) + xTx = feature.T * (weights * feature) + ws = xTx.I * (feature.T * (weights * label)) + predict[i] = feature[i, ] * ws + return predict + +if __name__ == "__main__": + # 1、导入数据集 + feature, label = load_data("data.txt") + predict = lwlr(feature, label, 0.002) + m = np.shape(predict)[0] + for i in range(m): + print feature[i, 1], predict[i] diff --git a/Chapter_8 RidgeRegression/ridge_regression_train.py b/Chapter_8 RidgeRegression/ridge_regression_train.py index f9a7810..157792c 100644 --- a/Chapter_8 RidgeRegression/ridge_regression_train.py +++ b/Chapter_8 RidgeRegression/ridge_regression_train.py @@ -1,216 +1,216 @@ -# coding:UTF-8 -''' -Date:20160928 -@author: zhaozhiyong -''' - -import numpy as np - -def load_data(file_path): - '''导入训练数据 - input: file_path(string):训练数据 - output: feature(mat):特征 - label(mat):标签 - ''' - f = open(file_path) - feature = [] - label = [] - for line in f.readlines(): - feature_tmp = [] - lines = line.strip().split("\t") - feature_tmp.append(1) # x0 - for i in xrange(len(lines) - 1): - feature_tmp.append(float(lines[i])) - feature.append(feature_tmp) - label.append(float(lines[-1])) - f.close() - return np.mat(feature), np.mat(label).T - -def ridge_regression(feature, label, lam): - '''最小二乘的求解方法 - input: feature(mat):特征 - label(mat):标签 - output: w(mat):回归系数 - ''' - n = np.shape(feature)[1] - w = (feature.T * feature + lam * np.mat(np.eye(n))).I * feature.T * label - return w - -def get_gradient(feature, label, w, lam): - '''计算导函数的值 - input: feature(mat):特征 - label(mat):标签 - output: w(mat):回归系数 - ''' - err = (label - feature * w).T - left = err * (-1) * feature - return left.T + lam * w - -def get_result(feature, label, w, lam): - ''' - input: feature(mat):特征 - label(mat):标签 - output: w(mat):回归系数 - ''' - left = (label - feature * w).T * (label - feature * w) - right = lam * w.T * w - return (left + right) / 2 - -def get_error(feature, label, w): - ''' - input: feature(mat):特征 - label(mat):标签 - output: w(mat):回归系数 - ''' - m = np.shape(feature)[0] - left = (label - feature * w).T * (label - feature * w) - return (left / (2 * m))[0, 0] - - -def bfgs(feature, label, lam, maxCycle): - '''利用bfgs训练Ridge Regression模型 - input: feature(mat):特征 - label(mat):标签 - lam(float):正则化参数 - maxCycle(int):最大迭代次数 - output: w(mat):回归系数 - ''' - n = np.shape(feature)[1] - # 1、初始化 - w0 = np.mat(np.zeros((n, 1))) - rho = 0.55 - sigma = 0.4 - Bk = np.eye(n) - k = 1 - while (k < maxCycle): - print "\titer: ", k, "\terror: ", get_error(feature, label, w0) - gk = get_gradient(feature, label, w0, lam) # 计算梯度 - dk = np.mat(-np.linalg.solve(Bk, gk)) - m = 0 - mk = 0 - while (m < 20): - newf = get_result(feature, label, (w0 + rho ** m * dk), lam) - oldf = get_result(feature, label, w0, lam) - if (newf < oldf + sigma * (rho ** m) * (gk.T * dk)[0, 0]): - mk = m - break - m = m + 1 - - # BFGS校正 - w = w0 + rho ** mk * dk - sk = w - w0 - yk = get_gradient(feature, label, w, lam) - gk - if (yk.T * sk > 0): - Bk = Bk - (Bk * sk * sk.T * Bk) / (sk.T * Bk * sk) + (yk * yk.T) / (yk.T * sk) - - k = k + 1 - w0 = w - return w0 - -def lbfgs(feature, label, lam, maxCycle, m=10): - '''利用lbfgs训练Ridge Regression模型 - input: feature(mat):特征 - label(mat):标签 - lam(float):正则化参数 - maxCycle(int):最大迭代次数 - m(int):lbfgs中选择保留的个数 - output: w(mat):回归系数 - ''' - n = np.shape(feature)[1] - # 1、初始化 - w0 = np.mat(np.zeros((n, 1))) - rho = 0.55 - sigma = 0.4 - - H0 = np.eye(n) - - s = [] - y = [] - - k = 1 - gk = get_gradient(feature, label, w0, lam) # 3X1 - print gk - dk = -H0 * gk - # 2、迭代 - while (k < maxCycle): - print "iter: ", k, "\terror: ", get_error(feature, label, w0) - m = 0 - mk = 0 - gk = get_gradient(feature, label, w0, lam) - # 2.1、Armijo线搜索 - while (m < 20): - newf = get_result(feature, label, (w0 + rho ** m * dk), lam) - oldf = get_result(feature, label, w0, lam) - if newf < oldf + sigma * (rho ** m) * (gk.T * dk)[0, 0]: - mk = m - break - m = m + 1 - - # 2.2、LBFGS校正 - w = w0 + rho ** mk * dk - - # 保留m个 - if k > m: - s.pop(0) - y.pop(0) - - # 保留最新的 - sk = w - w0 - qk = get_gradient(feature, label, w, lam) # 3X1 - yk = qk - gk - - s.append(sk) - y.append(yk) - - # two-loop - t = len(s) - a = [] - for i in xrange(t): - alpha = (s[t - i - 1].T * qk) / (y[t - i - 1].T * s[t - i - 1]) - qk = qk - alpha[0, 0] * y[t - i - 1] - a.append(alpha[0, 0]) - r = H0 * qk - - for i in xrange(t): - beta = (y[i].T * r) / (y[i].T * s[i]) - r = r + s[i] * (a[t - i - 1] - beta[0, 0]) - - if yk.T * sk > 0: - print "update OK!!!!" - dk = -r - - k = k + 1 - w0 = w - return w0 - -def save_weights(file_name, w0): - '''保存最终的结果 - input: file_name(string):需要保存的文件 - w0(mat):权重 - ''' - f_result = open("weights", "w") - m, n = np.shape(w0) - for i in xrange(m): - w_tmp = [] - for j in xrange(n): - w_tmp.append(str(w0[i, j])) - f_result.write("\t".join(w_tmp) + "\n") - f_result.close() - - -if __name__ == "__main__": - # 1、导入数据 - print "----------1.load data ------------" - feature, label = load_data("data.txt") - # 2、训练模型 - print "----------2.training ridge_regression ------------" - method = "lbfgs" # 选择的方法 - if method == "bfgs": # 选择BFGS训练模型 - w0 = bfgs(feature, label, 0.5, 1000) - elif method == "lbfgs": # 选择L-BFGS训练模型 - w0 = lbfgs(feature, label, 0.5, 1000, m=10) - else: # 使用最小二乘的方法 - w0 = ridge_regression(feature, label, 0.5) - # 3、保存最终的模型 - print "----------3.save model ------------" - save_weights("weights", w0) +# coding:UTF-8 +''' +Date:20160928 +@author: zhaozhiyong +''' + +import numpy as np + +def load_data(file_path): + '''导入训练数据 + input: file_path(string):训练数据 + output: feature(mat):特征 + label(mat):标签 + ''' + f = open(file_path) + feature = [] + label = [] + for line in f.readlines(): + feature_tmp = [] + lines = line.strip().split("\t") + feature_tmp.append(1) # x0 + for i in range(len(lines) - 1): + feature_tmp.append(float(lines[i])) + feature.append(feature_tmp) + label.append(float(lines[-1])) + f.close() + return np.mat(feature), np.mat(label).T + +def ridge_regression(feature, label, lam): + '''最小二乘的求解方法 + input: feature(mat):特征 + label(mat):标签 + output: w(mat):回归系数 + ''' + n = np.shape(feature)[1] + w = (feature.T * feature + lam * np.mat(np.eye(n))).I * feature.T * label + return w + +def get_gradient(feature, label, w, lam): + '''计算导函数的值 + input: feature(mat):特征 + label(mat):标签 + output: w(mat):回归系数 + ''' + err = (label - feature * w).T + left = err * (-1) * feature + return left.T + lam * w + +def get_result(feature, label, w, lam): + ''' + input: feature(mat):特征 + label(mat):标签 + output: w(mat):回归系数 + ''' + left = (label - feature * w).T * (label - feature * w) + right = lam * w.T * w + return (left + right) / 2 + +def get_error(feature, label, w): + ''' + input: feature(mat):特征 + label(mat):标签 + output: w(mat):回归系数 + ''' + m = np.shape(feature)[0] + left = (label - feature * w).T * (label - feature * w) + return (left / (2 * m))[0, 0] + + +def bfgs(feature, label, lam, maxCycle): + '''利用bfgs训练Ridge Regression模型 + input: feature(mat):特征 + label(mat):标签 + lam(float):正则化参数 + maxCycle(int):最大迭代次数 + output: w(mat):回归系数 + ''' + n = np.shape(feature)[1] + # 1、初始化 + w0 = np.mat(np.zeros((n, 1))) + rho = 0.55 + sigma = 0.4 + Bk = np.eye(n) + k = 1 + while (k < maxCycle): + print ("\titer: ", k, "\terror: ", get_error(feature, label, w0)) + gk = get_gradient(feature, label, w0, lam) # 计算梯度 + dk = np.mat(-np.linalg.solve(Bk, gk)) + m = 0 + mk = 0 + while (m < 20): + newf = get_result(feature, label, (w0 + rho ** m * dk), lam) + oldf = get_result(feature, label, w0, lam) + if (newf < oldf + sigma * (rho ** m) * (gk.T * dk)[0, 0]): + mk = m + break + m = m + 1 + + # BFGS校正 + w = w0 + rho ** mk * dk + sk = w - w0 + yk = get_gradient(feature, label, w, lam) - gk + if (yk.T * sk > 0): + Bk = Bk - (Bk * sk * sk.T * Bk) / (sk.T * Bk * sk) + (yk * yk.T) / (yk.T * sk) + + k = k + 1 + w0 = w + return w0 + +def lbfgs(feature, label, lam, maxCycle, m=10): + '''利用lbfgs训练Ridge Regression模型 + input: feature(mat):特征 + label(mat):标签 + lam(float):正则化参数 + maxCycle(int):最大迭代次数 + m(int):lbfgs中选择保留的个数 + output: w(mat):回归系数 + ''' + n = np.shape(feature)[1] + # 1、初始化 + w0 = np.mat(np.zeros((n, 1))) + rho = 0.55 + sigma = 0.4 + + H0 = np.eye(n) + + s = [] + y = [] + + k = 1 + gk = get_gradient(feature, label, w0, lam) # 3X1 + print (gk) + dk = -H0 * gk + # 2、迭代 + while (k < maxCycle): + print ("iter: ", k, "\terror: ", get_error(feature, label, w0)) + m = 0 + mk = 0 + gk = get_gradient(feature, label, w0, lam) + # 2.1、Armijo线搜索 + while (m < 20): + newf = get_result(feature, label, (w0 + rho ** m * dk), lam) + oldf = get_result(feature, label, w0, lam) + if newf < oldf + sigma * (rho ** m) * (gk.T * dk)[0, 0]: + mk = m + break + m = m + 1 + + # 2.2、LBFGS校正 + w = w0 + rho ** mk * dk + + # 保留m个 + if k > m: + s.pop(0)#remove and return item at index (default last). + y.pop(0) + + # 保留最新的 + sk = w - w0 + qk = get_gradient(feature, label, w, lam) # 3X1 + yk = qk - gk + + s.append(sk) + y.append(yk) + + # two-loop + t = len(s) + a = [] + for i in range(t): + alpha = (s[t - i - 1].T * qk) / (y[t - i - 1].T * s[t - i - 1]) + qk = qk - alpha[0, 0] * y[t - i - 1] + a.append(alpha[0, 0]) + r = H0 * qk + + for i in range(t): + beta = (y[i].T * r) / (y[i].T * s[i]) + r = r + s[i] * (a[t - i - 1] - beta[0, 0]) + + if yk.T * sk > 0: + print ("update OK!!!!") + dk = -r + + k = k + 1 + w0 = w + return w0 + +def save_weights(file_name, w0): + '''保存最终的结果 + input: file_name(string):需要保存的文件 + w0(mat):权重 + ''' + f_result = open("weights", "w") + m, n = np.shape(w0) + for i in range(m): + w_tmp = [] + for j in range(n): + w_tmp.append(str(w0[i, j])) + f_result.write("\t".join(w_tmp) + "\n") + f_result.close() + + +if __name__ == "__main__": + # 1、导入数据 + print ("----------1.load data ------------") + feature, label = load_data("data.txt") + # 2、训练模型 + print ("----------2.training ridge_regression ------------") + method = "lbfgs" # 选择的方法 + if method == "bfgs": # 选择BFGS训练模型 + w0 = bfgs(feature, label, 0.5, 1000) + elif method == "lbfgs": # 选择L-BFGS训练模型 + w0 = lbfgs(feature, label, 0.5, 1000, m=10) + else: # 使用最小二乘的方法 + w0 = ridge_regression(feature, label, 0.5) + # 3、保存最终的模型 + print ("----------3.save model ------------") + save_weights("weights", w0) From e664b94ab2cacaaf138b4715b3efe568e3e787c4 Mon Sep 17 00:00:00 2001 From: WangGitHubWei Date: Mon, 11 Jun 2018 16:02:34 +0800 Subject: [PATCH 10/10] 1 --- Chapter_4 SVM/svm_test.py | 162 +++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/Chapter_4 SVM/svm_test.py b/Chapter_4 SVM/svm_test.py index d02826e..01ec72d 100644 --- a/Chapter_4 SVM/svm_test.py +++ b/Chapter_4 SVM/svm_test.py @@ -1,81 +1,81 @@ -# coding:UTF-8 - -import numpy as np -import cPickle as pickle -from svm import svm_predict - -def load_test_data(test_file): - '''导入测试数据 - input: test_file(string):测试数据 - output: data(mat):测试样本的特征 - ''' - data = [] - f = open(test_file) - for line in f.readlines(): - lines = line.strip().split(' ') - - # 处理测试样本中的特征 - index = 0 - tmp = [] - for i in xrange(0, len(lines)): - li = lines[i].strip().split(":") - if int(li[0]) - 1 == index: - tmp.append(float(li[1])) - else: - while(int(li[0]) - 1 > index): - tmp.append(0) - index += 1 - tmp.append(float(li[1])) - index += 1 - while len(tmp) < 13: - tmp.append(0) - data.append(tmp) - f.close() - return np.mat(data) - -def load_svm_model(svm_model_file): - '''导入SVM模型 - input: svm_model_file(string):SVM模型保存的文件 - output: svm_model:SVM模型 - ''' - with open(svm_model_file, 'r') as f: - svm_model = pickle.load(f) - return svm_model - -def get_prediction(test_data, svm): - '''对样本进行预测 - input: test_data(mat):测试数据 - svm:SVM模型 - output: prediction(list):预测所属的类别 - ''' - m = np.shape(test_data)[0] - prediction = [] - for i in xrange(m): - # 对每一个样本得到预测值 - predict = svm_predict(svm, test_data[i, :]) - # 得到最终的预测类别 - prediction.append(str(np.sign(predict)[0, 0])) - return prediction - -def save_prediction(result_file, prediction): - '''保存预测的结果 - input: result_file(string):结果保存的文件 - prediction(list):预测的结果 - ''' - f = open(result_file, 'w') - f.write(" ".join(prediction)) - f.close() - -if __name__ == "__main__": - # 1、导入测试数据 - print "--------- 1.load data ---------" - test_data = load_test_data("svm_test_data") - # 2、导入SVM模型 - print "--------- 2.load model ----------" - svm_model = load_svm_model("model_file") - # 3、得到预测值 - print "--------- 3.get prediction ---------" - prediction = get_prediction(test_data, svm_model) - # 4、保存最终的预测值 - print "--------- 4.save result ----------" - save_prediction("result", prediction) +# coding:UTF-8 + +import numpy as np +import cPickle as pickle +from svm import svm_predict + +def load_test_data(test_file): + '''导入测试数据 + input: test_file(string):测试数据 + output: data(mat):测试样本的特征 + ''' + data = [] + f = open(test_file) + for line in f.readlines(): + lines = line.strip().split(' ') + + # 处理测试样本中的特征 + index = 0 + tmp = [] + for i in range(0, len(lines)): + li = lines[i].strip().split(":") + if int(li[0]) - 1 == index: + tmp.append(float(li[1])) + else: + while(int(li[0]) - 1 > index): + tmp.append(0) + index += 1 + tmp.append(float(li[1])) + index += 1 + while len(tmp) < 13: + tmp.append(0) + data.append(tmp) + f.close() + return np.mat(data) + +def load_svm_model(svm_model_file): + '''导入SVM模型 + input: svm_model_file(string):SVM模型保存的文件 + output: svm_model:SVM模型 + ''' + with open(svm_model_file, 'r') as f: + svm_model = pickle.load(f) + return svm_model + +def get_prediction(test_data, svm): + '''对样本进行预测 + input: test_data(mat):测试数据 + svm:SVM模型 + output: prediction(list):预测所属的类别 + ''' + m = np.shape(test_data)[0] + prediction = [] + for i in range(m): + # 对每一个样本得到预测值 + predict = svm_predict(svm, test_data[i, :]) + # 得到最终的预测类别 + prediction.append(str(np.sign(predict)[0, 0])) + return prediction + +def save_prediction(result_file, prediction): + '''保存预测的结果 + input: result_file(string):结果保存的文件 + prediction(list):预测的结果 + ''' + f = open(result_file, 'w') + f.write(" ".join(prediction)) + f.close() + +if __name__ == "__main__": + # 1、导入测试数据 + print ("--------- 1.load data ---------") + test_data = load_test_data("svm_test_data") + # 2、导入SVM模型 + print ("--------- 2.load model ----------") + svm_model = load_svm_model("model_file") + # 3、得到预测值 + print ("--------- 3.get prediction ---------") + prediction = get_prediction(test_data, svm_model) + # 4、保存最终的预测值 + print ("--------- 4.save result ----------") + save_prediction("result", prediction)