[Angelica/Steph] Refactored code for SVR/TDNN system.

riot_neuralnet/data.py → svr_plus_tdnn/data.py

@@ -5,28 +5,22 @@
 
 EMOTION_DIMENSION_COUNT = 4 # emotional dimensions: arousal, valence, expectation, power
 
-def get_delayed_emotion_training_data(time_delay=2):
-    raw_training_labels = get_raw_training_labels()
-    X_train, y_train = get_delayed_training_data(time_delay, range(1,18), raw_training_labels)
-    X_test, y_test = get_delayed_training_data(time_delay, range(18,21), raw_training_labels)
 
-    return (X_train, y_train, X_test, y_test)
-
-def get_delayed_training_data(time_delay, time_series_keys, raw_training_data):
-    X = list()
-    y = list()
-    for time_series_idx in time_series_keys:
-        time_series = raw_training_data[time_series_idx]
-        for image_idx in range(time_delay, len(time_series)):
-            data_point = [time_series[time_delay_idx] for time_delay_idx in range(image_idx-time_delay, image_idx)]
-            label = time_series[image_idx]
-            X.append([data_point])
-            y.append(label)
+def get_image_feature_vector_array():
+    feature = Feature()
+    root_directory = '../images'
+    features = list()
+    for subfile in os.listdir(root_directory):
+        if "DS_Store" in subfile:   continue
+        for file in os.listdir(root_directory + '/' + subfile):
+            if "DS_Store" not in file:
+                image_file = root_directory + '/' + subfile + '/' + file
+                features.append(feature.extract_hog_feature_vector(image_file)[0])
 
-    return (np.array(X), np.array(y))
+    return np.array(features)
 
 def get_raw_training_labels():
-    # Uses 10 photo series from the Cohn-Kanade dataset
+    # Uses 20 photo series from the Cohn-Kanade dataset
     # hand labeled by AP
     # arousal(least, most), valence(negative, positive), power, anticipation
     raw_training_labels = {1: [10, [.6, .4, .7, .6], [.9, .1, .8, .9]],
@@ -115,20 +109,6 @@ def get_image_feature_vector_batches():
     return features
 
 
-def get_image_feature_vector_array():
-    feature = Feature()
-    root_directory = '../images'
-    features = list()
-    for subfile in os.listdir(root_directory):
-        if "DS_Store" in subfile:   continue
-        for file in os.listdir(root_directory + '/' + subfile):
-            if "DS_Store" not in file:
-                image_file = root_directory + '/' + subfile + '/' + file
-                features.append(feature.extract_hog_feature_vector(image_file)[0])
-
-    return np.array(features)
-
-
 def get_shifted_training_labels(time_delay=2):
     raw_training_labels = get_raw_training_labels()
     shifted_training_labels = list()
@@ -142,3 +122,22 @@ def get_shifted_training_labels(time_delay=2):
 
     return np.array(final_labels)
 
+def get_delayed_emotion_training_data(time_delay=2):
+    raw_training_labels = get_raw_training_labels()
+    X_train, y_train = get_delayed_training_data(time_delay, range(1,18), raw_training_labels)
+    X_test, y_test = get_delayed_training_data(time_delay, range(18,21), raw_training_labels)
+
+    return (X_train, y_train, X_test, y_test)
+
+def get_delayed_training_data(time_delay, time_series_keys, raw_training_data):
+    X = list()
+    y = list()
+    for time_series_idx in time_series_keys:
+        time_series = raw_training_data[time_series_idx]
+        for image_idx in range(time_delay, len(time_series)):
+            data_point = [time_series[time_delay_idx] for time_delay_idx in range(image_idx-time_delay, image_idx)]
+            label = time_series[image_idx]
+            X.append([data_point])
+            y.append(label)
+
+    return (np.array(X), np.array(y))

svr_plus_tdnn/feature.py

@@ -0,0 +1,12 @@
+from skimage import color, io
+from skimage.feature import hog
+
+
+class Feature:
+
+    def extract_hog_feature_vector(self, imageFile, image_resize_length=64):
+        image = io.imread(imageFile)
+        image.resize((image_resize_length,image_resize_length))
+        image = color.rgb2gray(image)
+        featureVector, hog_image = hog(image, orientations=8, pixels_per_cell=(16, 16), cells_per_block=(1, 1), visualise=True)
+        return featureVector, hog_image

riot_neuralnet/main.py → svr_plus_tdnn/main.py

@@ -1,10 +1,11 @@
 from tdnn import TDNN
-from data import get_delayed_emotion_training_data
 from data import get_image_feature_vector_array, get_training_label_array, get_time_delay_training_data
 from regressionModel import RegressionModel
 
 
-def main():
+def main(verbose=False):
+
+    print 'Extracting features...'
     features = get_image_feature_vector_array()
     labels = get_training_label_array()
 
@@ -13,12 +14,14 @@ def main():
     model.fit()
     predictions = model.predict()
 
+    print 'Applying time-delay to regression output...'
     X_train, y_train, X_test, y_test = get_time_delay_training_data(predictions, predictions)
 
-    print ('X_train: ' + str(X_train.shape))
-    print ('y_train: ' + str(y_train.shape))
-    print('X_test: ' + str(X_test.shape))
-    print ('y_test: ' + str(y_test.shape))
+    if verbose:
+        print ('X_train: ' + str(X_train.shape))
+        print ('y_train: ' + str(y_train.shape))
+        print('X_test: ' + str(X_test.shape))
+        print ('y_test: ' + str(y_test.shape))
 
     print 'Training TDNN...'
     tdnn = TDNN(verbose=True)

svr_plus_tdnn/regressionModel.py

@@ -0,0 +1,25 @@
+from keras.layers import Dense
+from keras.models import Sequential
+import math
+
+class RegressionModel:
+    def __init__(self, features, labels, num_output_values=4, test_data_percentage=0.25):
+        self.test_data_percentage = test_data_percentage
+        self.features = features
+        self.labels = labels
+        self.test_features = features[int(math.ceil(len(features) * (1 - test_data_percentage))):len(features)]
+        self.test_labels = labels[int(math.ceil(len(labels) * (1 - test_data_percentage))):len(labels)]
+        self.feature_vector_length = len(features[0])
+
+        self.model = Sequential()
+        self.model.add(Dense(num_output_values, input_shape=(self.feature_vector_length,), activation='sigmoid'))
+
+    def fit(self):
+        self.model.compile(loss='mean_squared_error', optimizer='rmsprop', metrics=['accuracy'])
+
+        self.model.fit(self.features, self.labels,
+                  batch_size=1, epochs=2,
+                  validation_split=self.test_data_percentage)
+
+    def predict(self):
+        return self.model.predict(self.test_features, batch_size=1)

riot_neuralnet/tdnn.py → svr_plus_tdnn/tdnn.py

@@ -5,19 +5,19 @@
 
 class TDNN:
 
-    def __init__(self, verbose=False):
+    def __init__(self, time_delay=3, num_output_values=4, verbose=False):
         net = Sequential()
-        net.add(Conv2D(filters = 10, kernel_size=(3,4), activation='sigmoid', input_shape=(1,3,4), padding='same'))
+        net.add(Conv2D(filters=10, kernel_size=(time_delay, num_output_values), activation='sigmoid', input_shape=(1,time_delay,num_output_values), padding='same'))
         net.add(Flatten())
         net.add(Dense(units=4, activation='sigmoid'))
-        net.compile(optimizer='RMSProp', loss='cosine_proximity',
-                      metrics=['accuracy'])
         if verbose:
             net.summary()
 
         self.net = net
 
     def train(self, X_train, y_train, X_test, y_test):
+        self.net.compile(optimizer='RMSProp', loss='cosine_proximity',
+                    metrics=['accuracy'])
         self.net.fit(X_train, y_train, batch_size=10, epochs=100, validation_data=(X_test, y_test),
             callbacks=[ReduceLROnPlateau(), EarlyStopping(patience=3)])