Merge branch 'develop'

# Conflicts:
#	setup.py

.gitignore

@@ -1,4 +1,5 @@
 .idea/
 __pycache__
 dist/
-TensorImage.egg-info/
+TensorImage.egg-info/
+build/

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='TensorImage',
-    version='2.0.1',
+    version='2.0.2',
     description='Image classification library for easily and quickly deploying models and training classifiers',
     long_description=long_description,
     author='TensorImage',

tensorimage/__init__.py

@@ -1,22 +1,16 @@
-__all__ = ["train", "util", "image", "file", "data_augmentation", "classify", "config", "make_workspace"]
+__all__ = ["base", "train", "util", "image", "file", "data_augmentation", "classify", "config", "dataset"]
 
-from tensorimage.util.make_workspace import make_workspace
-from tensorimage.util.mkdir import mkdir
-from tensorimage.util.make_file import make_json_file
-from tensorimage.util.convnet_builder import ConvNetBuilder
-from tensorimage.train.display_architecture import display_architecture
-from tensorimage.train.l2_regularization import L2RegularizationBuilder
-from tensorimage.train.models import RosNet
-from tensorimage.train.models import AlexNet
-from tensorimage.train.ops import conv2d
-from tensorimage.train.ops import maxpool2d
+from tensorimage.util.system.make_workspace import make_workspace
+from tensorimage.util.system.mkdir import mkdir
+from tensorimage.util.system.make_file import make_json_file
+from tensorimage.base.display_architecture import display_architecture
+from tensorimage.base.l2_regularization import L2RegularizationBuilder
+from tensorimage.base.ops import conv2d
+from tensorimage.base.ops import maxpool2d
+from tensorimage.base.weights_initializer import init_weights
 from tensorimage.train.trainer import Trainer
-from tensorimage.train.trainer import ClusterTrainer
-from tensorimage.train.weights_initializer import init_weights
+from tensorimage.train.cluster_trainer import ClusterTrainer
 from tensorimage.image.display import display_image
-from tensorimage.image.label_path_writer import write_unclassified_dataset_paths
-from tensorimage.image.label_path_writer import write_labels
-from tensorimage.image.label_path_writer import write_training_dataset_paths
 from tensorimage.image.loader import ImageLoader
 from tensorimage.image.writer import DataWriter
 from tensorimage.image.writer import TrainingDataWriter
@@ -37,6 +31,9 @@
 from tensorimage.config.info import base_unclassified_data_store_path
 from tensorimage.config.info import base_predictions_store_path
 from tensorimage.config.info import base_trained_models_store_path
+from tensorimage.base.models.map.model import model_map
+from tensorimage.base.models import rosnet
+from tensorimage.base.models import alexnet
 from tensorimage.config.manager import set_config
 from tensorimage.config.manager import view_config
 from tensorimage.data_augmentation.ops import AddPepperSaltNoise

tensorimage/base/__init__.py

@@ -0,0 +1 @@
+__all__ = ["display_architecture", "l2_regularization", "ops", "weights_initializer", "models"]

tensorimage/train/l2_regularization.py → tensorimage/base/l2_regularization.py

@@ -1,5 +1,5 @@
 import tensorflow as tf
-from tensorimage.train.weights_initializer import init_weights
+from tensorimage.base.weights_initializer import init_weights
 
 
 class L2RegularizationBuilder:

tensorimage/base/models/__init__.py

@@ -0,0 +1 @@
+__all__ = ["alexnet", "rosnet", "map"]

tensorimage/train/models.py → tensorimage/base/models/alexnet.py

@@ -1,71 +1,6 @@
-from tensorimage.train.ops import *
-from tensorimage.train.weights_initializer import init_weights
-from tensorimage.train.display_architecture import display_architecture
-
-
-class RosNet:
-    """rosnet"""
-    def __init__(self, x, n_classes):
-        self.x = x
-        self.n_classes = n_classes
-
-        height, width = self.x.shape[1], self.x.shape[2]
-        self.shape = [1, height, width, 3]
-        self.rv = self.get_rv(self.shape)
-
-        self.weights_shapes = {
-            "conv1": [5, 5, 3, 3],
-            "conv2": [3, 3, 3, 64],
-            "fcl": [self.rv*64, 128],
-            "out": [128, self.n_classes]
-        }
-        self.biases_shapes = {
-            "conv1": [3],
-            "conv2": [64],
-            "fcl": [128],
-            "out": [self.n_classes]
-        }
-
-    def convnet(self):
-        with tf.name_scope('rosnet'):
-            with tf.name_scope('conv1_layer'):
-                with tf.name_scope('conv1'):
-                    conv1 = conv2d(self.x, init_weights('weights', 'conv1', [5, 5, 3, 3]) +
-                                   init_weights('biases', 'conv1', [3]))
-                with tf.name_scope('conv1_maxpool2d'):
-                    conv1_maxpool2d = maxpool2d(conv1)
-            with tf.name_scope('conv2_layer'):
-                with tf.name_scope('conv2'):
-                    conv2 = conv2d(conv1_maxpool2d, init_weights('weights', 'conv2', [3, 3, 3, 64]) +
-                                   init_weights('biases', 'conv2', [64]))
-                with tf.name_scope('conv2_maxpool2d'):
-                    conv2_maxpool2d = maxpool2d(conv2)
-            with tf.name_scope('fcl_layer'):
-                with tf.name_scope('flatten'):
-                    fclr = tf.reshape(conv2_maxpool2d, [tf.shape(self.x)[0], self.rv*64])
-                with tf.name_scope('ReLU_add_matmul'):
-                    fcl = tf.nn.relu(tf.add(tf.matmul(fclr, init_weights('weights', 'fcl', [self.rv*64, 128])),
-                                            init_weights('biases', 'fcl', [128])))
-            with tf.name_scope('out_layer'):
-                model = tf.add(tf.matmul(fcl, init_weights('weights', 'out', [128, self.n_classes])),
-                               init_weights('biases', 'out', [self.n_classes]))
-
-        display_architecture(Conv1=conv1.shape,
-                             Conv1_MaxPool2d=conv1_maxpool2d.shape,
-                             Conv2=conv2.shape,
-                             Conv2_MaxPool2d=conv2_maxpool2d.shape,
-                             FCL_flatten=fclr.shape,
-                             FCL=fcl.shape,
-                             OutputLayer=model.shape)
-        return model
-
-    @staticmethod
-    def get_rv(shape):
-        conv1 = conv2d(tf.ones(shape), init_weights('rv', 'conv1', [5, 5, 3, 3]))
-        conv1_maxpool2d = maxpool2d(conv1)
-        conv2 = conv2d(conv1_maxpool2d, init_weights('rv', 'conv2', [3, 3, 3, 64]))
-        conv2_maxpool2d = maxpool2d(conv2)
-        return conv2_maxpool2d.shape[1]*conv2_maxpool2d.shape[2]
+from tensorimage.base.ops import *
+from tensorimage.base.weights_initializer import init_weights
+from tensorimage.base.display_architecture import display_architecture
 
 
 class AlexNet:

tensorimage/base/models/map/__init__.py

@@ -0,0 +1 @@
+__all__ = ["model"]

tensorimage/base/models/map/model.py

@@ -0,0 +1,6 @@
+from tensorimage.base.models import alexnet
+from tensorimage.base.models import rosnet
+
+
+model_map = {'rosnet': rosnet.RosNet,
+             'alexnet': alexnet.AlexNet}

tensorimage/base/models/rosnet.py

@@ -0,0 +1,70 @@
+from tensorimage.base.ops import *
+from tensorimage.base.weights_initializer import init_weights
+from tensorimage.base.display_architecture import display_architecture
+
+
+class RosNet:
+    """rosnet"""
+    def __init__(self, x, n_classes):
+        self.x = x
+        self.n_classes = n_classes
+
+        height, width = self.x.shape[1], self.x.shape[2]
+        self.shape = [1, height, width, 3]
+        self.rv = self.get_rv(self.shape)
+
+        self.layer_names = ["conv1", "conv2", "fcl", "out"]
+
+        self.weights_shapes = {
+            self.layer_names[0]: [5, 5, 3, 3],
+            self.layer_names[1]: [3, 3, 3, 64],
+            self.layer_names[2]: [self.rv*64, 128],
+            self.layer_names[3]: [128, self.n_classes]
+        }
+        self.biases_shapes = {
+            "conv1": [3],
+            "conv2": [64],
+            "fcl": [128],
+            "out": [self.n_classes]
+        }
+
+    def convnet(self):
+        with tf.name_scope('rosnet'):
+            with tf.name_scope('conv1_layer'):
+                with tf.name_scope('conv1'):
+                    conv1 = conv2d(self.x, init_weights('weights', 'conv1', [5, 5, 3, 3]) +
+                                   init_weights('biases', 'conv1', [3]))
+                with tf.name_scope('conv1_maxpool2d'):
+                    conv1_maxpool2d = maxpool2d(conv1)
+            with tf.name_scope('conv2_layer'):
+                with tf.name_scope('conv2'):
+                    conv2 = conv2d(conv1_maxpool2d, init_weights('weights', 'conv2', [3, 3, 3, 64]) +
+                                   init_weights('biases', 'conv2', [64]))
+                with tf.name_scope('conv2_maxpool2d'):
+                    conv2_maxpool2d = maxpool2d(conv2)
+            with tf.name_scope('fcl_layer'):
+                with tf.name_scope('flatten'):
+                    fclr = tf.reshape(conv2_maxpool2d, [tf.shape(self.x)[0], self.rv*64])
+                with tf.name_scope('ReLU_add_matmul'):
+                    fcl = tf.nn.relu(tf.add(tf.matmul(fclr, init_weights('weights', 'fcl', [self.rv*64, 128])),
+                                            init_weights('biases', 'fcl', [128])))
+            with tf.name_scope('out_layer'):
+                model = tf.add(tf.matmul(fcl, init_weights('weights', 'out', [128, self.n_classes])),
+                               init_weights('biases', 'out', [self.n_classes]))
+
+        display_architecture(Conv1=conv1.shape,
+                             Conv1_MaxPool2d=conv1_maxpool2d.shape,
+                             Conv2=conv2.shape,
+                             Conv2_MaxPool2d=conv2_maxpool2d.shape,
+                             FCL_flatten=fclr.shape,
+                             FCL=fcl.shape,
+                             OutputLayer=model.shape)
+        return model
+
+    @staticmethod
+    def get_rv(shape):
+        conv1 = conv2d(tf.ones(shape), init_weights('rv', 'conv1', [5, 5, 3, 3]))
+        conv1_maxpool2d = maxpool2d(conv1)
+        conv2 = conv2d(conv1_maxpool2d, init_weights('rv', 'conv2', [3, 3, 3, 64]))
+        conv2_maxpool2d = maxpool2d(conv2)
+        return conv2_maxpool2d.shape[1]*conv2_maxpool2d.shape[2]

tensorimage/classify/__init__.py

@@ -1 +1 @@
-__all__ = ["restore_model"]
+__all__ = ["classifier", "restore_model"]

tensorimage/classify/classifier.py

@@ -1,13 +1,14 @@
 import tensorflow as tf
 import numpy as np
 import csv
+import ast
 
 from tensorimage.config.info import *
 from tensorimage.image.display import display_image
 from tensorimage.file.reader import *
-from tensorimage.util.mkdir import mkdir
-from tensorimage.util.convnet_builder import ConvNetBuilder
+from tensorimage.util.system.mkdir import mkdir
 from tensorimage.classify.restore_model import ModelRestorer
+from tensorimage.base.models.map.model import model_map
 
 
 class Classifier:
@@ -30,7 +31,7 @@ def __init__(self,
         self.training_name = training_name
         self.classification_name = classification_name
         try:
-            self.show_images = eval(str(show_images[0]))
+            self.show_images = ast.literal_eval(str(show_images[0]))
             self.max_images = show_images[1]
         except NameError:
             self.show_images = False
@@ -49,7 +50,7 @@ def __init__(self,
         self.height = image_metadata["height"]
         self.path_file = image_metadata["path_file"]
         self.trainable = image_metadata["trainable"]
-        if eval(self.trainable):
+        if ast.literal_eval(self.trainable):
             raise AssertionError("Data is trainable")
 
         # Read trained model metadata
@@ -85,11 +86,8 @@ def __init__(self,
         self.X = None
         self.X_ = None
 
-        self.convnet_builder = ConvNetBuilder(self.architecture)
-        self.convolutional_neural_network = self.convnet_builder.build_convnet()
-
         self.sess = tf.Session()
-        self.model_restore = ModelRestorer(self.model_path, self.model_name, self.architecture, self.sess)
+        self.model_restorer = ModelRestorer(self.model_path, self.model_name, self.architecture, self.sess)
 
         self.n_images = 0
 
@@ -100,11 +98,11 @@ def build_dataset(self):
         self.n_images = len(self.X)
 
     def predict(self):
-        self.model_restore.start()
+        self.model_restorer.start()
 
         x = tf.placeholder(tf.float32, [None, self.height, self.width, 3])
 
-        convnet = self.convolutional_neural_network(x, self.n_classes)
+        convnet = model_map[self.architecture](x, self.n_classes)
         model = convnet.convnet()
         self.sess.run(tf.global_variables_initializer())
 

tensorimage/classify/restore_model.py

@@ -1,7 +1,8 @@
 import numpy as np
 import tensorflow as tf
 from tensorimage.config.info import workspace_dir
-from tensorimage.train.weights_initializer import init_weights
+from tensorimage.base.weights_initializer import init_weights
+from tensorimage.base.models.map.model import model_map
 
 
 class ModelRestorer:
@@ -11,41 +12,20 @@ def __init__(self, model_folder_name: str, model_name: str, architecture: str, s
         self.architecture = architecture
         self.sess = sess
 
-    def start(self):
-        if self.architecture == 'RosNet':
-            self.restore_rosnet_model()
-        elif self.architecture == 'AlexNet':
-            self.restore_alexnet_model()
+        self.layer_names = model_map[self.architecture].layer_names
 
-    def restore_rosnet_model(self):
+    def start(self):
         saver = tf.train.import_meta_graph(workspace_dir + 'user/trained_models/' + self.model_folder_name + '/' + self.model_name + '.meta')
         saver.restore(self.sess, tf.train.latest_checkpoint(workspace_dir + 'user/trained_models/' + self.model_folder_name + '/./'))
 
-        layers = ['conv1', 'conv2', 'fcl', 'out']
-        with tf.variable_scope('RosNet', reuse=tf.AUTO_REUSE):
+        with tf.variable_scope(self.architecture, reuse=tf.AUTO_REUSE):
             with tf.name_scope('weights_restore'):
-                for layer in layers:
-                    ly = self.sess.run('weights/'+layer+':0')
-                    ly_list = np.ndarray.tolist(ly)
-                    init_weights('weights', layer, ly.shape, initializer=tf.initializers.constant(ly_list))
+                for layer in self.layer_names:
+                    layer_weights = self.sess.run('weights/' + layer + ':0')
+                    layer_weights_ = np.ndarray.tolist(layer_weights)
+                    init_weights('weights', layer, layer_weights.shape, initializer=tf.initializers.constant(layer_weights_))
             with tf.name_scope('biases_restore'):
-                for layer in layers:
-                    ly = self.sess.run('biases/'+layer+':0')
-                    ly_list = np.ndarray.tolist(ly)
-                    init_weights('biases', layer, ly.shape, initializer=tf.initializers.constant(ly_list))
-
-    def restore_alexnet_model(self):
-        saver = tf.train.import_meta_graph(workspace_dir + 'user/trained_models/' + self.model_folder_name + '/' + self.model_name + '.meta')
-        saver.restore(self.sess, tf.train.latest_checkpoint(workspace_dir + 'user/trained_models/' + self.model_folder_name + '/./'))
-
-        layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fcl', 'fcl2', 'out']
-        with tf.name_scope('weights_restore'):
-            for layer in layers:
-                ly = self.sess.run('weights/'+layer+':0')
-                ly_list = np.ndarray.tolist(ly)
-                init_weights('weights', layer, ly.shape, initializer=tf.initializers.constant(ly_list))
-        with tf.name_scope('biases_restore'):
-            for layer in layers:
-                ly = self.sess.run('biases/'+layer+':0')
-                ly_list = np.ndarray.tolist(ly)
-                init_weights('biases', layer, ly.shape, initializer=tf.initializers.constant(ly_list))
+                for layer in self.layer_names:
+                    layer_biases = self.sess.run('biases/' + layer + ':0')
+                    layer_biases_ = np.ndarray.tolist(layer_biases)
+                    init_weights('biases', layer, layer_biases.shape, initializer=tf.initializers.constant(layer_biases_))

tensorimage/config/__init__.py

@@ -0,0 +1 @@
+__all__ = ["info", "manager"]

tensorimage/data_augmentation/src.py

@@ -20,8 +20,10 @@ def __init__(self, x, y, verbose, n_classes: int, n_channels=3):
         self.n_classes = n_classes
         self.n_channels = n_channels
 
-        assert len(self.x.shape) == 4 and len(self.y.shape) == 2
-        assert self.x.shape[0] == self.y.shape[0]
+        if not (len(self.x.shape) == 4 and len(self.y.shape) == 2):
+            raise AssertionError()
+        if not self.x.shape[0] == self.y.shape[0]:
+            raise AssertionError()
 
         self.n_images = self.x.shape[0]
         self.sess = tf.Session()
@@ -36,7 +38,7 @@ def flip(self, dims):
         log.info("Image flipping", self)
         augmented_data = tf.constant([], tf.float32, shape=[0, dims[0], dims[1], self.n_channels])
         augmented_labels = tf.constant([], tf.float32, shape=[0, self.n_classes])
-        for (image_n, image), (label_n, label) in zip(enumerate(self.x), enumerate(self.y)):
+        for (image_n, image), label in zip(enumerate(self.x), self.y):
             flip_up_down = tf.image.flip_up_down(image)
             flip_left_right = tf.image.flip_left_right(image)
             random_flip_up_down = tf.image.random_flip_up_down(image)

tensorimage/dataset/__init__.py

@@ -0,0 +1 @@
+__all__ = ["label_writer", "path_writer"]