cvpr submit final

luyongxi · Nov 16, 2016 · 5286384 · 5286384
1 parent e2c1a16
commit 5286384
Show file tree

Hide file tree

Showing 13 changed files with 155 additions and 91 deletions.
diff --git a/experiments/scripts/ccc_baseline_single_celeba_submit.sh b/experiments/scripts/ccc_baseline_single_celeba_submit.sh
@@ -5,7 +5,7 @@ do
     jbsub -mem 10g -mail -cores 4+1 -queue x86_short ./train_baseline_single_celeba.sh 10000 8000 0.001 small32-lowvgg16 0 ${i}
 done
 
-# for i in `seq 0 39`;
-# do
-#     jbsub -mem 10g -mail -cores 4+1 -queue x86 ./train_baseline_single_celeba.sh 10000 8000 0.001 lowvgg16 0 ${i}
-# done
+for i in `seq 0 39`;
+do
+    jbsub -mem 10g -mail -cores 4+1 -queue x86 ./train_baseline_single_celeba.sh 10000 8000 0.001 lowvgg16 0 ${i}
+done
diff --git a/experiments/scripts/ccc_submit_baselines.sh b/experiments/scripts/ccc_submit_baselines.sh
@@ -1,16 +1,16 @@
 #!/bin/bash
 
-# experiments with baseline using thin models
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 60000 20000 0.001 small32-lowvgg16 0
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0
+experiments with baseline using thin models
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 60000 20000 0.001 small32-lowvgg16 0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0
 
-# experiments with baseline using thin models (from scratch)
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_scratch_celeba.sh 60000 20000 0.001 small32-lowvgg16 0
+experiments with baseline using thin models (from scratch)
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_scratch_celeba.sh 60000 20000 0.001 small32-lowvgg16 0
 
-# # experiments with baseline using large models
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 40000 16000 0.001 lowvgg16 0
+# experiments with baseline using large models
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 40000 16000 0.001 lowvgg16 0
 jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 40000 16000 0.001 lowvgg16 0
 
-# # experiments with baseline using low rank models
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 40000 16000 0.001 lowvgg16 16
-jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 40000 16000 0.001 lowvgg16 16
+# experiments with baseline using low rank models
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_celeba.sh 40000 16000 0.001 lowvgg16 16
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 40000 16000 0.001 lowvgg16 16
diff --git a/experiments/scripts/ccc_submit_branching.sh b/experiments/scripts/ccc_submit_branching.sh
@@ -8,13 +8,18 @@
 # jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_celeba.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 1.0
 # jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_celeba.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 0.0
 # jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_celeba.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 0.5
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_celeba.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 2.0
 
 
 # # experiments with branching on DeepFashion
-jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 0.0
-jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 1.0
-jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 2.0
+# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 0.0
+# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 1.0
+# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 2.0
+# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 2.0
+
 
 # # experiments with joint dataset
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small32-lowvgg16 0 15 2000 1.0
-# jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small32-lowvgg16 0 15 2000 2.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small64-lowvgg16 0 15 2000 1.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small64-lowvgg16 0 15 2000 2.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small32-lowvgg16 0 15 2000 1.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_person.sh 100000 40000 0.001 small32-lowvgg16 0 15 2000 2.0
diff --git a/experiments/scripts/ccc_submit_deepfashion.sh b/experiments/scripts/ccc_submit_deepfashion.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# submit deepfashion experiments
+
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_baseline_deepfashion.sh 40000 16000 0.001 lowvgg16 16
+
+# # experiments with branching on DeepFashion
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 0.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 1.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 2.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 3.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small32-lowvgg16 0 15 1000 4.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 0.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 0.5
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 1.0
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_branch_deepfashion.sh 60000 20000 0.001 small64-lowvgg16 0 15 1000 2.0
diff --git a/experiments/scripts/ccc_submit_regroup.sh b/experiments/scripts/ccc_submit_regroup.sh
@@ -3,4 +3,5 @@
 jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt original [19,31,18,26,36,4,13,14,17,38,15,16,22,30,1,8,27,29,33,34,37,3,7,39,23,32,21,25,6,5,10,20,2,9,28,0,12,24,11,35]
 jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt rand1 [27,20,26,17,25,28,18,13,23,14,33,37,6,2,12,22,11,24,5,3,35,39,21,31,15,1,34,30,8,38,10,29,32,36,19,16,7,9,4,0]
 jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt rand2 [31,26,20,5,35,11,28,25,21,2,34,6,3,22,9,17,37,27,29,33,30,8,24,16,32,0,14,19,39,13,12,15,18,4,10,1,38,23,7,36]
-jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt rand3 [3,6,31,28,1,36,8,18,35,21,17,2,15,10,9,12,0,4,33,11,16,37,24,20,38,39,30,14,29,32,7,22,19,27,5,13,26,25,34,23]
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt rand3 [3,6,31,28,1,36,8,18,35,21,17,2,15,10,9,12,0,4,33,11,16,37,24,20,38,39,30,14,29,32,7,22,19,27,5,13,26,25,34,23]
+jbsub -mem 10g -cores 4+1 -queue x86 -mail ./train_prototxt.sh 60000 output/celeba_small32-lowvgg16_0_15_1000_2.0/celeba_train/prototxt/round_11/solver.prototxt inverse [37,3,7,39,23,32,21,25,6,5,10,20,2,9,28,0,12,24,11,35,19,31,18,26,36,4,13,14,17,38,15,16,22,30,1,8,27,29,33,34]
diff --git a/experiments/scripts/comet_submit_deepfashion.sh b/experiments/scripts/comet_submit_deepfashion.sh
diff --git a/experiments/scripts/train_branch_person.sh b/experiments/scripts/train_branch_person.sh
@@ -22,17 +22,17 @@ cd ../..
 
 time ./tools/load_person.py --gpu 0 \
     --model_face output/celeba_baseline_lowvgg16_0/celeba_train/prototxt/test.prototxt \
-    --weights_face output/celeba_baseline_lowvgg16_0/celeba_train/celeba_train_iter_40000.caffemodel \
+    --weights_face output/celeba_baseline_lowvgg16_0/celeba_train/deploy.caffemodel \
     --model_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/prototxt/test.prototxt \
-    --weights_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/deepfashion_train_iter_40000.caffemodel \
+    --weights_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/deploy.caffemodel \
     --imdb_face person_clothes_train \
     --imdb_clothes person_face_train
 
 time ./tools/load_person.py --gpu 0 \
     --model_face output/celeba_baseline_lowvgg16_0/celeba_train/prototxt/test.prototxt \
-    --weights_face output/celeba_baseline_lowvgg16_0/celeba_train/celeba_train_iter_40000.caffemodel \
+    --weights_face output/celeba_baseline_lowvgg16_0/celeba_train/deploy.caffemodel \
     --model_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/prototxt/test.prototxt \
-    --weights_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/deepfashion_train_iter_40000.caffemodel \
+    --weights_clothes output/deepfashion_baseline_lowvgg16_0/deepfashion_train/deploy.caffemodel \
     --imdb_face person_clothes_val \
     --imdb_clothes person_face_val
 

diff --git a/lib/datasets/deepfashion.py b/lib/datasets/deepfashion.py
@@ -129,7 +129,8 @@ def evaluate(self, scores, ind, cls_idx=None):
         if cls_idx is None:
             cls_idx = np.arange(self.num_classes)
 
-        gt = self.gtdb['attr'][ind, cls_idx]
+        gt = self.gtdb['attr'][ind, :]
+        gt = gt[:, cls_idx]
         err = compute_mle(scores, gt)
 
         return err

diff --git a/lib/datasets/personattr.py b/lib/datasets/personattr.py
@@ -19,7 +19,11 @@ class PersonAttributes(Imdb):
 
     def __init__(self, split, align=False, partition='all'):
 
-        name = 'person_' + partition + '_' + split
+        if not partition == 'all':
+            name = 'person_' + partition + '_' + split
+        elif partition == 'all':
+            name = 'person_' + split
+
         if align and (partition == 'all' or partition == 'face'):
             name += '_align'
 
@@ -65,9 +69,12 @@ def _load_dataset(self, split, align, partition):
             else:
                 fn = osp.join(self.data_path, 'person_'+'face'+'_'+split+'.pkl') 
             if osp.exists(fn):
-                with open(fn, 'rb') as fid:
-                    labels = cPickle.load(fid)
-                    self._gtdb['attr'][:celeba_num, self._clothes_class_idx] = labels
+                if partition == 'all':
+                    with open(fn, 'rb') as fid:
+                        labels = cPickle.load(fid)
+                        self._gtdb['attr'][:celeba_num, self._clothes_class_idx] = labels
+                else:
+                    'Dataset {}: Labels for clothes attributes on CelebA are not loaded, the partition is not "all"'.format(self.name)
             else:
                 print 'Dataset {}: Labels for clothes attributes on CelebA are not available! Missing filename: {}. Did you forget to run load_person.py first?'.\
                     format(self.name, fn)
@@ -78,9 +85,12 @@ def _load_dataset(self, split, align, partition):
             # load soft labels for face attributes on deepfashion
             fn = osp.join(self.data_path, 'person_'+'clothes'+'_'+split+'.pkl')
             if osp.exists(fn):
-                with open(fn, 'rb') as fid:
-                    labels = cPickle.load(fid)
-                    self._gtdb['attr'][celeba_num:, self._face_class_idx] = labels
+                if partition == 'all':
+                    with open(fn, 'rb') as fid:
+                        labels = cPickle.load(fid)
+                        self._gtdb['attr'][celeba_num:, self._face_class_idx] = labels
+                else:
+                    'Dataset {}: Labels for face attributes on Deepfashion are not loaded, the partition is not "all"'.format(self.name)
             else:
                 print 'Dataset {}: Labels for face attributes on Deepfashion are not available! Missing filename: {}. Did you forget to run load_person.py first?'.\
                     format(self.name, fn)

diff --git a/lib/evaluation/test.py b/lib/evaluation/test.py
@@ -17,14 +17,18 @@ def test_cls_topk(net, imdb, cls_idx, k):
 
     if cls_idx is None:
         cls_idx = np.arange(imdb.num_classes)
+    else:
+        cls_idx = np.array(cls_idx)
 
     num_classes = len(cls_idx)
     assert k<=num_classes, 'k={} should be less than or equal to num_classes={}'.\
         format(k, num_classes)
     num_images = imdb.num_images
 
     # iterate over images, collect error vectors
-    found = np.zeros((num_images, num_classes)) # in {0,1} format
+    # only test attributes that are properly labelled
+    val_ind = np.any(imdb.gtdb['attr'][:, cls_idx]>=0, axis=0)
+    found = np.zeros((num_images, np.sum(val_ind))) # in {0,1} format
     timer = Timer()
     for i in xrange(num_images):
         # prepare blobs 
@@ -39,8 +43,8 @@ def test_cls_topk(net, imdb, cls_idx, k):
         # get results
         scores = blobs_out[label_name]
         # find recall of top-k attributes
-        top_classes = np.argsort(-scores)[0, :k]
-        pos_classes = np.where(imdb.gtdb['attr'][i, cls_idx] == 1)[0]
+        top_classes = np.argsort(-scores[:, val_ind])[0, :k]
+        pos_classes = np.where(imdb.gtdb['attr'][i, cls_idx[val_ind]] == 1)[0]
 
         found_classes =  [idx for idx in pos_classes if idx in top_classes]
         found[i, found_classes] = 1.0
@@ -53,31 +57,37 @@ def test_cls_topk(net, imdb, cls_idx, k):
     print '!!! Summary of results.'
     print '!!! Test model on the "{}" dataset'.format(imdb.name)
     print '!!! The dataset has {} images.'.format(imdb.num_images)
-    print '!!! On average, there are {} active attribute classese per image'.format(np.mean(np.sum(imdb.gtdb['attr'][:, cls_idx], axis=1)))
+    print '!!! On average, there are {} active attribute classese per image'.format(np.mean(np.sum(imdb.gtdb['attr'][:, cls_idx[val_ind]], axis=1)))
     print '!!! The average run time is {} per image.'.format(timer.average_time)
 
-    # get error for each class
+    # get recall for each class
     class_names = imdb.classes
-    recall = np.nansum(found, axis=0)/np.sum(imdb.gtdb['attr'][:, cls_idx], axis=0)
-    for i in xrange(len(cls_idx)):
-        print '!!! Top {} recall rate for class {} is: {}'.\
-            format(k, class_names[cls_idx[i]], recall[i])
+    num_pos = np.sum(imdb.gtdb['attr'][:, cls_idx[val_ind]], axis=0)
+    recall = np.nansum(found, axis=0)/num_pos
+    for i in xrange(np.sum(val_ind)):
+        print '!!! Top {} recall rate for class {} is: {} ({} instances)'.\
+            format(k, class_names[cls_idx[val_ind][i]], recall[i], num_pos[i])
 
-    print '!!! The top-{} recall rate is {}.'.format(k, recall.mean())
+    print '!!! The top-{} recall rate is {}.'.format(k, np.nanmean(recall))
     print '---------------------------------------------------------------'
 
+    # print top-k accuracy
+    print '!!! The top-{} accuracy rate is {}.'.format(k, np.sum(np.any(found>0, axis=1), dtype=np.float64)/num_images)
 
 def test_cls_error(net, imdb, cls_idx):
     """ Test a model on imdb and evaluate the error rate. """
 
     if cls_idx is None:
         cls_idx = np.arange(imdb.num_classes)
+    else:
+        cls_idx = np.array(cls_idx)
 
     num_classes = len(cls_idx)
     num_images = imdb.num_images
 
     # iterate over images, collect error vectors
-    err = np.zeros((num_images, num_classes)) # in {0,1} format
+    val_ind = np.any(imdb.gtdb['attr'][:, cls_idx]>=0, axis=0)
+    err = np.zeros((num_images, np.sum(val_ind))) # in {0,1} format
     timer = Timer()
     for i in xrange(num_images):
         # prepare blobs 
@@ -92,7 +102,7 @@ def test_cls_error(net, imdb, cls_idx):
         # get results
         scores = blobs_out[label_name]
         # evaluate the scores
-        err[i,:] = imdb.evaluate(scores, np.array([i]), cls_idx)
+        err[i,:] = imdb.evaluate(scores[:, val_ind], np.array([i]), cls_idx[val_ind])
         # print infos
         print 'Image {}/{} ::: speed: {:.3f}s per image.'.format(i, num_images, timer.average_time)
 
@@ -106,9 +116,9 @@ def test_cls_error(net, imdb, cls_idx):
     # get error for each class
     class_names = imdb.classes
     mean_err = np.nanmean(err, axis=0)
-    for i in xrange(len(cls_idx)):
+    for i in xrange(np.sum(val_ind)):
         print '!!! Error rate for class {} is: {}'.\
-            format(class_names[cls_idx[i]], mean_err[i])
+            format(class_names[cls_idx[val_ind][i]], mean_err[i])
 
     print '!!! The average error rate is {}.'.format(mean_err.mean())
     print '---------------------------------------------------------------'

diff --git a/lib/models/factory.py b/lib/models/factory.py
@@ -94,10 +94,6 @@ def get_io(name, **kwargs):
     path='.', **kwargs: 
     _sp_low_vgg_16_gen(io=io, model_name=model_name, **kwargs))
 
-__models['sp1-lowvgg16'] = (lambda io, model_name='sp1-lowvgg16', 
-    path='.', **kwargs: 
-    _sp1_low_vgg_16_gen(io=io, model_name=model_name, **kwargs))
-
 def get_models(name, **kwargs):
     """ Get an model (network architecture) by name."""
     if not __models.has_key(name):
@@ -240,36 +236,13 @@ def _sp_low_vgg_16_gen(io, model_name, **kwargs):
                     'fc':{13:64,14:64}, 'output': {15:16}}
 
     # specify number of outputs
-    num_outputs = {'conv':{0:32,1:64,2:128,3:128,4:224,5:224,6:224,
-                           7:224,8:256,9:256,10:256,11:288,12:288}, 
-                    'fc':{13:576,14:768}}
+    num_outputs = {'conv':{0:32,1:32,2:32,3:32,4:96,5:96,6:96,
+                           7:192,8:256,9:288,10:288,11:288,12:416}, 
+                    'fc':{13:896,14:1216}}
 
     return _low_vgg_16_base(io, model_name, 
         num_filters=num_filters, num_outputs=num_outputs, include_dropout=[], **kwargs)
 
-def _sp1_low_vgg_16_gen(io, model_name, **kwargs):
-    """ 
-        The VGG-16 variants with smaller number of outputs at each layer,
-        and the model is full rank
-
-        We also turn off dropout.
-
-        VGG-16 -> 13 fully connected layers and 2 fully connected layers 
-        last_low_rank specifies the last layer to perform low-rank factorization. 
-    """
-    # parameters for convolutional layers
-    num_filters = {'conv':{0:8,1:8,2:16,3:16,4:32,5:32,6:32,
-                           7:32,8:32,9:32,10:32,11:32,12:32}, 
-                    'fc':{13:64,14:64}, 'output': {15:16}}
-
-    # specify number of outputs
-    num_outputs = {'conv':{0:32,1:32,2:32,3:32,4:32,5:32,6:32,
-                           7:64,8:64,9:160,10:160,11:320,12:320}, 
-                    'fc':{13:1280,14:1280}}
-
-    return _low_vgg_16_base(io, model_name, 
-        num_filters=num_filters, num_outputs=num_outputs, include_dropout=[], **kwargs)   
-
 def _low_vgg_16_base(io, model_name, **kwargs):
     """ VGG-16 -> 13 fully connected layers and 2 fully connected layers 
         last_low_rank specifies the first layer to perform low-rank factorization.