In [1]:
#加载所需的模块
from sklearn import datasets
from matplotlib import pyplot as plt
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from matplotlib.colors import ListedColormap
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FuncAnimation
import copy
%matplotlib notebook
In [2]:
#生成样本
X,y= datasets.make_circles(n_samples = 2000, factor=0.3, noise=.1)
#划分训练集和测试集
X, X_test, y, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
#构建特征空间
# c,r = np.mgrid[[slice(X.min() - .5,X.max() + .5,50j)]*2]
c,r = np.mgrid[[slice(X.min() - .5,X.max() + .5,50j)]*2]
p = np.c_[c.flat,r.flat]
#归一化
#归一化的两种方式：各自归一化，还是根据样本的分布进行归一化？
ss = StandardScaler().fit(X)
X = ss.transform(X)
p = ss.transform(p)
X_test = ss.transform(X_test)
#可视化
fig = plt.figure(figsize = (9,3))
cm_bright = ListedColormap(['#f59d38','#1d84c4'])
plt.subplot(121)
m1 = plt.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
plt.title('train samples')
plt.axis('equal')
plt.subplot(122)
m2 = plt.scatter(*X_test.T,c = y_test,cmap = 'bwr',edgecolors='white');
plt.title('test samples')
plt.axis('equal')
ax = fig.get_axes()
plt.colorbar(ax = ax);
plt.show();
保存生成的图片
# plt.savefig('C:/Users/ASUS/Desktop/sample.jpg')
In [20]:
#训练神经网络，进行1000次迭代，并将每次迭代的结果存储下来
w,b,z,loss,train,test = [],[],[],[], [], []
MLP = MLPClassifier(hidden_layer_sizes=(3,2),max_iter = 20,warm_start = True)
for i in range(1000):
MLP.fit(X,y)
ww = copy.deepcopy(MLP.coefs_)
w.append(ww)
bb = copy.deepcopy(MLP.intercepts_)
b.append(bb)
z.append(MLP.predict_proba(p))
loss.append(MLP.loss_)
train.append(MLP.score(X,y))
test.append(MLP.score(X_test,y_test))
...;
# 定义sigmoid函数
def sigmoid(x):
return(1/(1+np.exp(-x)))
X_first_sigmoid = []
p_first_sigmoid = []
X_second_linear = []
p_second_linear = []
X_second_sigmoid = []
p_second_sigmoid = []
X_third_linear = []
p_third_linear = []
X_third_sigmoid = []
p_third_sigmoid = []
# 保存前200次迭代，每一层的线性变换值以及进行sigmoid非线性变换后的值
for n in range(200):
X_first_l = X @ w[n][0] + b[n][0]
X_first_s = sigmoid(X_first_l)
p_first_l = p @ w[n][0] + b[n][0]
p_first_s = sigmoid(p_first_l)
X_second_l = X_first_s @ w[n][1] + b[n][1]
p_second_l = p_first_s @ w[n][1] + b[n][1]
X_second_s = sigmoid(X_second_l)
p_second_s = sigmoid(p_second_l)
X_third_l = X_second_s @ w[n][2] + b[n][2]
p_third_l = p_second_s @ w[n][2] + b[n][2]
X_third_s = sigmoid(X_third_l)
p_third_s = sigmoid(p_third_l)
X_first_sigmoid.append(X_first_s)
p_first_sigmoid.append(p_first_s)
X_second_linear.append(X_second_l)
X_second_sigmoid.append(X_second_s)
X_third_linear.append(X_third_l)
X_third_sigmoid.append(X_third_s)
p_second_linear.append(p_second_l)
p_second_sigmoid.append(p_second_s)
p_third_linear.append(p_third_l)
p_third_sigmoid.append(p_third_s)
# 结果分析
fig,((ax1,ax2),(ax3,ax4)) = plt.subplots(2,2, figsize=(6, 6))
ax1.scatter(*p.T,c = z[-1][:,1],cmap = 'PiYG')
ax1.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
ax1.set_title('train score:%.5f'%train[-1])
ax2.scatter(*p.T,c = z[-1][:,1],cmap = 'PiYG')
ax2.scatter(*X_test.T,c = y_test,cmap = 'bwr',edgecolors='white')
ax2.set_title('test score:%.5f'%test[-1])
ax3.plot(loss)
ax3.set_title('Loss')
ax4.plot(train,c = 'r',label = 'train')
ax4.plot(test,c = 'b',label = 'test')
ax4.set_title('train-test score');
ax4.legend();
plt.show();
In [41]:
# 表现总览
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, axs = plt.subplots(1,2,figsize = (6,3))
def update(n):
fig.suptitle(str(n))
axs[0].clear()
axs[0].scatter(*p.T,c = z[n][:,1],cmap = 'PiYG')
axs[0].scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white',alpha = 0.1)
axs[0].set_title('train score:%.5f'%train[n])
axs[0].axis('off')
axs[1].clear()
axs[1].scatter(*p.T,c = z[n][:,1],cmap = 'PiYG')
axs[1].scatter(*X_test.T,c = y_test,cmap = 'bwr',edgecolors='white',alpha = 0.1)
axs[1].set_title('test score:%.5f'%test[n])
axs[1].axis('off')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# 保存图片的代码，下面也是一样的（因为保存图片要花比较长的时间，所以把代码注释起来了）
# ani.save('overview.gif', writer='imagemagick')
In [8]:
# 第一隐层变换总览
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig,((axs1,axs2),(axs3,axs4)) = plt.subplots(2,2,figsize = (8,8),subplot_kw = {'projection':'3d'})
def update(n):
layer_one_split_linear = np.where(p @ w[n][0] + b[n][0].T<=0,0,1)
layer_one_split_result_linear = layer_one_split_linear[:,0]*1+layer_one_split_linear[:,1]*2+layer_one_split_linear[:,2]*4
layer_one_split_sigmoid = np.where(sigmoid(p @ w[n][0] + b[n][0])<=0.5,0.5,1)
layer_one_split_result_sigmoid = layer_one_split_sigmoid[:,0]*1+layer_one_split_sigmoid[:,1]*2+layer_one_split_sigmoid[:,2]*4
fig.suptitle(str(n))
axs1.clear()
axs1.set_title('First Hidden Layer: Linear Transform')
axs1.scatter(*(X @ w[n][0] + b[n][0]).T,c = y,cmap = 'bwr',edgecolors='white')
axs1.elev,ax1.azim = 15, -60
axs2.clear()
axs2.set_title('First Hidden Layer: Linear Transform')
axs2.scatter(*(p @ w[n][0] + b[n][0]).T,c = layer_one_split_result_linear,cmap = 'tab20',edgecolors='white')
axs2.elev,ax1.azim = 15, -60
axs3.clear()
axs3.set_title('First Hidden Layer: Sigmoid Transform')
axs3.scatter(*(1/(1 + np.exp(-(X @ w[n][0] + b[n][0])))).T,c = y,cmap = 'bwr',edgecolors='white')
axs3.elev,ax1.azim = 15, -60
axs4.clear()
axs4.set_title('First Hidden Layer: Sigmoid Transform')
axs4.scatter(*(1/(1 + np.exp(-(p @ w[n][0] + b[n][0])))).T,c = layer_one_split_result_sigmoid,cmap = 'tab20',edgecolors='white')
axs4.elev,ax1.azim = 15, -60
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_first_layer_overview.gif', writer='imagemagick')
In [9]:
# 第一隐层各节点变换
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, ((axs1,axs2,axs3),(axs4,axs5,axs6)) = plt.subplots(2,3,figsize = (12,8))
def update(n):
fig.suptitle(str(n))
axs1.clear()
axs1.set_title('First Neuron: Linear Transform')
axs1.scatter(*p.T,c = (p @ w[n][0][:,0] + b[n][0][0]),cmap = 'PiYG')
axs1.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs1.axis('off')
axs2.clear()
axs2.set_title('Second Neuron: Linear Transform')
axs2.scatter(*p.T,c = (p @ w[n][0][:,1] + b[n][0][1]),cmap = 'PiYG')
axs2.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs2.axis('off')
axs3.clear()
axs3.set_title('Third Neuron: Linear Transform')
axs3.scatter(*p.T,c = (p @ w[n][0][:,2] + b[n][0][2]),cmap = 'PiYG')
axs3.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs3.axis('off')
axs4.clear()
axs4.set_title('First Neuron: Sigmoid Transform')
axs4.scatter(*p.T,c = (1/(1+np.exp(-(p @ w[n][0][:,0] + b[n][0][0])))),cmap = 'PiYG')
axs4.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs4.axis('off')
axs5.clear()
axs5.set_title('Second Neuron: Sigmoid Transform')
axs5.scatter(*p.T,c = (1/(1+np.exp(-(p @ w[n][0][:,1] + b[n][0][1])))),cmap = 'PiYG')
axs5.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs5.axis('off')
axs6.clear()
axs6.set_title('Third Neuron: Sigmoid Transform')
axs6.scatter(*p.T,c = (1/(1+np.exp(-(p @ w[n][0][:,2] + b[n][0][2])))),cmap = 'PiYG')
axs6.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white')
axs6.axis('off')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/20, save_count = 100)
# ani.save('pre_first_layer_neuron.gif', writer='imagemagick')
In [42]:
# 第一隐层对输入层空间的划分
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, axs = plt.subplots(1,1,figsize = (4,4))
def update(n):
fig.suptitle(str(n))
layer_one_split_linear = np.where(p @ w[n][0] + b[n][0].T<=0,0,1)
layer_one_split_result_linear = layer_one_split_linear[:,0]*1+layer_one_split_linear[:,1]*2+layer_one_split_linear[:,2]*4
axs.clear()
axs.set_title('First Layer: Space Partition')
axs.scatter(*p.T,c = layer_one_split_result_linear,cmap = 'tab20')
axs.scatter(*X.T,c = y,cmap = 'bwr',edgecolors='white',alpha = 0.1)
axs.axis('off')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/20, save_count = 100)
# ani.save('pre_first_layer_divide.gif', writer='imagemagick')
In [43]:
# 第二隐层变换总览
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig,((axs1,axs2),(axs3,axs4)) = plt.subplots(2,2,figsize = (8,8))
def update(n):
fig.suptitle(str(n))
layer_two_split_linear = np.where(p_second_linear[n]<=0,0,1)
layer_two_split_result_linear = layer_two_split_linear[:,0]*1+layer_two_split_linear[:,1]*2
layer_two_split_sigmoid = np.where(p_second_sigmoid[n]<=0.5,0.5,1)
layer_two_split_result_sigmoid = layer_two_split_sigmoid[:,0]*1+layer_two_split_sigmoid[:,1]*2
axs1.clear()
axs1.set_title('Second Hidden Layer: Linear Transform')
axs1.scatter(*X_second_linear[n].T,c = y,cmap = 'bwr',edgecolors='white')
axs2.clear()
axs2.set_title('Second Hidden Layer: Linear Transform')
axs2.scatter(*p_second_linear[n].T,c = layer_two_split_result_linear,cmap = 'tab20',edgecolors='white')
axs3.clear()
axs3.set_title('Second Hidden Layer: Sigmoid Transform')
axs3.scatter(*sigmoid(X_second_linear[n]).T,c = y,cmap = 'bwr',edgecolors='white')
axs4.clear()
axs4.set_title('Second Hidden Layer: Sigmoid Transform')
axs4.scatter(*sigmoid(p_second_linear[n]).T,c = layer_two_split_result_sigmoid,cmap = 'tab20',edgecolors='white')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_second_layer_overview3.0.gif', writer='imagemagick')
In [15]:
# 第二隐层各节点变换
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, ((axs1,axs2),(axs4,axs5)) = plt.subplots(2,2,figsize = (12,12),subplot_kw = {'projection':'3d'})
def update(n):
fig.suptitle(str(n))
axs1.clear()
axs1.set_title('First Neuron: Linear Transform')
axs1.scatter(*p_first_sigmoid[n].T,c = p_second_linear[n][:,0],cmap = 'PiYG')
axs2.clear()
axs2.set_title('Second Neuron: Linear Transform')
axs2.scatter(*p_first_sigmoid[n].T,c = p_second_linear[n][:,1],cmap = 'PiYG')
axs4.clear()
axs4.set_title('First Neuron: Sigmoid Transform')
axs4.scatter(*p_first_sigmoid[n].T,c = p_second_sigmoid[n][:,0],cmap = 'PiYG')
axs5.clear()
axs5.set_title('Second Neuron: Sigmoid Transform')
axs5.scatter(*p_first_sigmoid[n].T,c = p_second_sigmoid[n][:,1],cmap = 'PiYG')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/20, save_count = 100)
# ani.save('pre_second_layer_neuron.gif', writer='imagemagick')
In [23]:
# 第二隐层对第一隐层空间的划分
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, axs = plt.subplots(1,1,figsize = (8,8),subplot_kw = {'projection':'3d'})
def update(n):
fig.suptitle(str(n))
layer_two_split_linear = np.where(p_second_linear[n]<=0,0,1)
layer_two_split_result_linear = layer_two_split_linear[:,0]*1+layer_two_split_linear[:,1]*2
axs.clear()
axs.set_title('Second Layer: Space Partition')
axs.scatter(*p_first_sigmoid[n].T,c = layer_two_split_result_linear,cmap = 'tab20c')
axs.scatter(*X_first_sigmoid[n].T,c = y,cmap = 'bwr',edgecolors='white')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_second_layer_divide3.0.gif', writer='imagemagick')
In [45]:
# 输出层取值的直方图，因为看不出什么信息来，所以我没有把结果往ppt里面放
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig,((axs1,axs2),(axs3,axs4)) = plt.subplots(2,2,figsize = (8,8))
def update(n):
fig.suptitle(str(n))
axs1.clear()
axs1.set_title('Output Layer: Linear Transform')
axs1.hist(X_third_linear[n])
axs2.clear()
axs2.set_title('Output Layer: Linear Transform')
axs2.hist(p_third_linear[n])
axs3.clear()
axs3.set_title('Output Layer: Sigmoid Transform')
axs3.hist(sigmoid(X_third_linear[n]))
axs4.clear()
axs4.set_title('Output Layer: Sigmoid Transform')
axs4.hist(sigmoid(p_third_linear[n]))
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_output_layer_hist.gif', writer='imagemagick')
In [44]:
# 每个神经元w取值的平方，通过这个值来看每个连接的重要性
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig,((axs1,axs2),(axs3,axs4),(axs5,axs6)) = plt.subplots(3,2,figsize = (6,6))
def update(n):
fig.suptitle(str(n))
axs1.clear()
axs1.set_title('First Layer')
axs1.barh(range(2), np.square(w[n][0][:,0]), height=0.7, color='steelblue', alpha=0.8)
axs1.set_xlim(0,np.max(w[n][0]))
axs1.set_ylabel("First Neuron")
axs2.clear()
axs2.set_title('Second Layer')
axs2.barh(range(3), np.square(w[n][1][:,0]), height=0.7, color='steelblue', alpha=0.8)
axs2.set_xlim(0,np.max(w[n][1]))
axs2.set_ylabel("First Neuron")
axs3.clear()
axs3.barh(range(2), np.square(w[n][0][:,1]), height=0.7, color='steelblue', alpha=0.8)
axs3.set_xlim(0,np.max(w[n][0]))
axs3.set_ylabel("Second Neuron")
axs4.clear()
axs4.barh(range(3), np.square(w[n][1][:,1]), height=0.7, color='steelblue', alpha=0.8)
axs4.set_xlim(0,np.max(w[n][1]))
axs4.set_ylabel("Second Neuron")
axs5.clear()
axs5.barh(range(2), np.square(w[n][0][:,2]), height=0.7, color='steelblue', alpha=0.8)
axs5.set_xlim(0,np.max(w[n][0]))
axs5.set_ylabel("Third Neuron")
axs6.axis('off')
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_weight.gif', writer='imagemagick')
In [45]:
# 输出层对第二隐层空间的划分
%matplotlib notebook
from matplotlib.animation import FuncAnimation, writers
fig, axs = plt.subplots(1,1,figsize = (8,8))
def update(n):
fig.suptitle(str(n))
layer_three_split_linear = np.where(p_third_linear[n]<=0,0,1)
layer_three_split_result_linear = layer_three_split_linear[:,0]*1
axs.clear()
axs.set_title('Output Layer: Space Partition and Prediction')
axs.scatter(*p_second_sigmoid[n].T,c =p_third_sigmoid[1][:,0],cmap = 'PiYG')
axs.scatter(*sigmoid(X_second_linear[n]).T,c = y,cmap = 'bwr',edgecolors='white',alpha = 0.1)
ani = FuncAnimation(fig, update, frames = range(200), interval = 1000.0/12, save_count = 100)
# ani.save('pre_output_layer_divide.gif', writer='imagemagick')
