Xiaowuhu/1405 (#228)

* update

* update

B-教学案例与实践/B6-神经网络基本原理简明教程/14.0-卷积神经网络.md

@@ -3,4 +3,9 @@ Copyright © Microsoft Corporation. All rights reserved.
 
 # 卷积神经网络
 
+特点 用途
 用于特征压缩
+
+举例 图片效果
+
+典型网络讲解 如LeNet, AlexNet, ResNet等 VGG16

B-教学案例与实践/B6-神经网络基本原理简明教程/14.4-卷积的实现.md

@@ -0,0 +1,28 @@
+Copyright © Microsoft Corporation. All rights reserved.
+  适用于[License](https：//github.com/Microsoft/ai-edu/blob/master/LICENSE.md)版权许可
+
+# 卷积的实现
+
+Python, for
+
+img2col
+
+numba
+
+gpu
+
+
+输入数据
+
+N：样本图片数量（比如一次计算10张图片）
+C：图片通道数量（比如红绿蓝三通道）
+H：图片高度（比如224）
+W：图片宽度（比如224）
+
+定义卷积Weights
+
+K：卷积核的数量（等于输出通道数）
+C：输入图片通道数
+FH：过滤器高度
+FW：过滤器宽度
+

B-教学案例与实践/B6-神经网络基本原理简明教程/微软-方案1/NeuralNetwork/ch14/MiniFramework.py

@@ -0,0 +1,67 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+from MiniFramework.NeuralNet import *
+from MiniFramework.GDOptimizer import *
+from MiniFramework.LossFunction import *
+from MiniFramework.Parameters import *
+from MiniFramework.WeightsBias import *
+from MiniFramework.Activators import *
+
+from MnistDataReader import *
+
+'''
+train_image_file = 'train-images-01'
+train_label_file = 'train-labels-01'
+test_image_file = 'test-images-01'
+test_label_file = 'test-labels-01'
+'''
+train_image_file = 'train-images-10'
+train_label_file = 'train-labels-10'
+test_image_file = 'test-images-10'
+test_label_file = 'test-labels-10'
+
+
+def LoadData(num_output):
+    mdr = MnistDataReader(train_image_file, train_label_file, test_image_file, test_label_file)
+    mdr.ReadData()
+    mdr.Normalize()
+    return mdr
+
+
+if __name__ == '__main__':
+
+    num_output = 10
+    dataReader = LoadData(num_output)
+    num_feature = dataReader.num_feature
+    num_example = dataReader.num_example
+    num_input = num_feature
+    num_hidden1 = 64
+    num_hidden2 = 32
+    max_epoch = 1
+    batch_size = 5
+    learning_rate = 0.02
+    eps = 0.01
+
+    params = CParameters(learning_rate, max_epoch, batch_size, eps,
+                        LossFunctionName.CrossEntropy3, 
+                        InitialMethod.Xavier, 
+                        OptimizerName.SGD)
+
+    loss_history = CLossHistory()
+
+    net = NeuralNet(params)
+    fc1 = FcLayer(num_input, num_hidden1, Relu())
+    net.add_layer(fc1, "fc1")
+    fc2 = FcLayer(num_hidden1, num_hidden2, Relu())
+    net.add_layer(fc2, "fc2")
+    fc3 = FcLayer(num_hidden2, num_output, Softmax())
+    net.add_layer(fc3, "fc3")
+    net.train(dataReader, loss_history)
+
+    loss_history.ShowLossHistory(params, 0, None, 0, 1)
+
+    net.load_parameters()
+    print("Testing...")
+    correct, count = net.Test(dataReader)
+    print(str.format("rate={0} / {1} = {2}", correct, count, correct/count))

B-教学案例与实践/B6-神经网络基本原理简明教程/微软-方案1/NeuralNetwork/ch14/MiniFramework/Activators.py

@@ -0,0 +1,68 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import numpy as np
+
+class CActivator(object):
+    # z = 本层的wx+b计算值矩阵
+    def forward(self, z):
+        pass
+
+    # z = 本层的wx+b计算值矩阵
+    # a = 本层的激活函数输出值矩阵
+    # delta = 上（后）层反传回来的梯度值矩阵
+    def backward(self, z, a, delta):
+        pass
+
+
+# 直传函数，相当于无激活
+class Identity(CActivator):
+    def forward(self, z):
+        return z
+
+    def backward(self, z, a, delta):
+        return delta, a
+
+
+class Sigmoid(CActivator):
+    def forward(self, z):
+        a = 1.0 / (1.0 + np.exp(-z))
+        return a
+
+    def backward(self, z, a, delta):
+        da = np.multiply(a, 1-a)
+        dz = np.multiply(delta, da)
+        return dz, da
+
+
+class Tanh(CActivator):
+    def forward(self, z):
+        a = 2.0 / (1.0 + np.exp(-2*z)) - 1.0
+        return a
+
+    def backward(self, z, a, delta):
+        da = 1 - np.multiply(a, a)
+        dz = np.multiply(delta, da)
+        return dz, da
+
+
+class Relu(CActivator):
+    def forward(self, z):
+        a = np.maximum(z, 0)
+        return a
+
+    # 注意relu函数判断是否大于1的根据是正向的wx+b=z的值，而不是a值
+    def backward(self, z, a, delta):
+        da = np.zeros(z.shape)
+        da[z>0] = 1
+        dz = da * delta
+        return dz, da
+
+
+class Softmax(CActivator):
+    def forward(self, z):
+        shift_z = z - np.max(z, axis=0)
+        exp_z = np.exp(shift_z)
+        a = exp_z / np.sum(exp_z, axis=0)
+        return a
+

B-教学案例与实践/B6-神经网络基本原理简明教程/微软-方案1/NeuralNetwork/ch14/MiniFramework/ConvLayer.py

@@ -0,0 +1,57 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#coding=utf-8
+
+import numpy as np
+
+from MiniFramework.Layer import *
+from MiniFramework.Activators import *
+from MiniFramework.ConvWeightsBias import *
+from MiniFramework.Parameters import *
+
+class ConvLayer(CLayer):
+    def __init__(self, num_input_channel, num_output_channel, num_filter_size, stride, padding, activator):
+        self.num_input_channel = num_input_channel
+        self.num_output_channel = num_output_channel
+        self.num_filter_size = num_filter_size
+        self.stride = stride
+        self.padding = padding
+        self.activator = activator
+
+    def Initialize(self):
+        self.weights = ConvWeightsBias(self.num_output_channel, self.num_input_channel, self.num_filter_size, self.num_filter_size)
+        self.weights.Initialize();
+
+    """
+    输入数据
+    N：样本图片数量（比如一次计算10张图片）
+    C：图片通道数量（比如红绿蓝三通道）
+    H：图片高度（比如224）
+    W：图片宽度（比如224）
+    思维卷积操作
+    """
+
+    def forward(self, x):
+        self.input_shape = x.shape
+        assert(x.ndim == 4)
+        self.x = x
+
+
+
+
+    # 把激活函数算做是当前层，上一层的误差传入后，先经过激活函数的导数，而得到本层的针对z值的误差
+    def backward(self, delta_in, flag):
+
+
+    def pre_update(self):
+        self.weights.pre_Update()
+
+    def update(self):
+        self.weights.Update()
+
+    def save_parameters(self, name):
+        self.weights.SaveResultValue(name)
+
+    def load_parameters(self, name):
+        self.weights.LoadResultValue(name)

B-教学案例与实践/B6-神经网络基本原理简明教程/微软-方案1/NeuralNetwork/ch14/MiniFramework/ConvWeightsBias.py

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#coding=utf-8
+
+import numpy as np
+
+"""
+Weights and Bias: 一个Weights可以包含多个卷积核Kernal，一个卷积核可以包含多个过滤器Filter
+WK - Kernal 卷积核数量(等于输出通道数量), 每个WK有一个Bias
+WC - Channel 输入通道数量
+FH - Filter Height
+FW - Filter Width
+"""
+class ConvWeightsBias(object):
+    def __init__(self, num_Kernal, num_Channel, num_Height, num_Width):
+        self.WK = num_Kernal
+        self.WC = num_Channel
+        self.FH = num_Height
+        self.FW = num_Width
+
+    def Initialize():
+        self.W = np.zeros((self.WK, self.WC, self.FH, self.FW))
+        self.B = np.zeros((self.WK, 1))
+
+
+if __name__ == '__main__':
+    wb = ConvWeightsBias(2,3,5,5)

B-教学案例与实践/B6-神经网络基本原理简明教程/微软-方案1/NeuralNetwork/ch14/MiniFramework/DataReader.py

@@ -0,0 +1,135 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import numpy as np
+from pathlib import Path
+from enum import Enum
+
+# we assume data format looks like:
+# example-> 1, 2, 3, 4
+# feature1  x  x  x  x
+# feature2  x  x  x  x
+#-------------------------
+# label     y  y  y  y
+
+
+class DataNormalization(Enum):
+    NormalizeX = 1,
+    NormalizeY = 2,
+    NormalizePredicate = 3
+
+class DataReader(object):
+    def __init__(self, x_file_name, y_file_name):
+        self.x_file_name = x_file_name
+        self.y_file_name = y_file_name
+        self.num_example = -1
+        self.num_feature = -1
+        self.num_category = -1
+
+    # read data from file
+    def ReadData(self):
+        Xfile = Path(self.x_file_name)
+        Yfile = Path(self.y_file_name)
+        if Xfile.exists() and Yfile.exists():
+            self.XRawData = np.load(Xfile)
+            self.YRawData = np.load(Yfile)
+
+            self.num_example = self.XRawData.shape[1]
+            self.num_feature = self.XRawData.shape[0]
+            self.num_category = len(np.unique(self.YRawData))
+
+            return self.XRawData, self.YRawData
+        # end if
+        return None,None
+
+    # normalize data by extracting range from source data
+    # return: X_new: normalized data with same shape
+    # return: X_norm: 2xN (features)
+    #               [[min1, min2, min3...]
+    #                [range1, range2, range3...]]
+
+    def NormalizeX(self):
+        self.X = np.zeros(self.XRawData.shape)
+        num_feature = self.XRawData.shape[0]
+        self.X_norm = np.zeros((2,num_feature))
+        # 按行归一化,即所有样本的同一特征值分别做归一化
+        for i in range(num_feature):
+            # get one feature from all examples
+            x = self.XRawData[i,:]
+            max_value = np.max(x)
+            min_value = np.min(x)
+            # min value
+            self.X_norm[0,i] = min_value 
+            # range value
+            self.X_norm[1,i] = max_value - min_value 
+            x_new = (x - self.X_norm[0,i]) / self.X_norm[1,i]
+            self.X[i,:] = x_new
+        # end for
+        return self.X
+
+    def NormalizeY(self):
+        self.Y = self.YRawData
+        return self.Y
+
+    # normalize data by specified range and min_value
+    def NormalizePredicateData(self, X_predicate):
+        X_new = np.zeros(X_predicate.shape)
+        n_feature = X_predicate.shape[0]
+        for i in range(n_feature):
+            x = X_predicate[i,:]
+            X_new[i,:] = (x-self.X_norm[0,i])/self.X_norm[1,i]
+        return X_new
+
+    # 获得批样本数据
+    def GetBatchSamples(self, batch_size, iteration):
+        start = iteration * batch_size
+        end = start + batch_size
+        batch_X = self.X[0:self.num_feature, start:end].reshape(self.num_feature, batch_size)
+        batch_Y = self.Y[:, start:end].reshape(-1, batch_size)
+        return batch_X, batch_Y
+
+    def ToOneHot(self):
+        self.Y = np.zeros((self.num_category, self.num_example))
+        for i in range(self.num_example):
+            if self.YRawData[0,i] == 1:
+                self.Y[0,i] = 1
+            elif self.YRawData[0,i] == 2:
+                self.Y[1,i] = 1
+            elif self.YRawData[0,i] == 3:
+                self.Y[2,i] = 1
+            # end if
+        # end for
+        return self.Y
+
+    # if use tanh function, need to set negative_value = -1
+    def ToZeroOne(YData, positive_label, negative_label, positiva_value = 1, negative_value = 0):
+        self.Y = np.zeros((1, self.num_example))
+        for i in range(self.num_example):
+            if YData[0,i] == negative_label:     # 负类的标签设为0
+                self.Y[0,i] = negative_value
+            elif YData[0,i] == positive_label:   # 正类的标签设为1
+                self.Y[0,i] = positiva_value
+            # end if
+        # end for
+        return self.Y
+
+    # permutation only affect along the first axis, so we need transpose the array first
+    # see the comment of this class to understand the data format
+    def Shuffle(self):
+        seed = np.random.randint(0,100)
+        np.random.seed(seed)
+        XP = np.random.permutation(self.X.T)
+        np.random.seed(seed)
+        YP = np.random.permutation(self.Y.T)
+        self.X = XP.T
+        self.Y = YP.T
+        return self.X, self.Y
+
+# unit test
+if __name__ == '__main__':
+    X = np.array([1,2,3,4,5,6,7,8]).reshape(2,4)
+    Y = np.array([7,8,9,0])
+    print(X,Y)
+    dp = DataReader()
+    X,Y=dp.Shuffle(X,Y)
+    print(X,Y)