miniimagenet-deepcluster and mnist-acai

.gitignore

@@ -94,7 +94,7 @@ ENV/
 
 # other
 data
-data.tar.gz
+*.tar.gz
 log
 models/
 

data_generator.py

@@ -316,14 +316,17 @@ def make_data_tensor(self, train=True):
                              'gtgt': 'class_ind'}[mode]
             for file_path, info in tqdm(miniimagenet_path_to_info_dict.items()):
                 partition[info[class_ind_key]].append(file_path_to_ind[file_path])
-            ipdb.set_trace()
+            for cls, indices in partition.items():
+                partition[cls] = indices[:600]
             partitions.append(partition)
         print('Number of partitions: {}'.format(len(partitions)))
         print('Average number of clusters/classes: {}'.format(np.mean([len(partition.keys()) for partition in partitions])))
 
         def sample_task():
             if mode == 'semi':
-                p = [0.8, 0.2]  # 80% k-means task, 20% miniimagenet ground truth
+                assert len(partitions) == 2
+                assert 0 <= FLAGS.p_gtgt <= 1
+                p = [1 - FLAGS.p_gtgt, FLAGS.p_gtgt]
             else:
                 p = None
             while True:

main.py

@@ -74,9 +74,11 @@
 flags.DEFINE_integer('num_res_blocks', 5, 'number of resnet blocks')
 flags.DEFINE_integer('num_parts_per_res_block', 2, 'number of bn-relu-conv parts in a res block')
 flags.DEFINE_bool('miniimagenet_only', False, 'only miniimagenet classes')
+flags.DEFINE_float('p_gtgt', 0.0, 'probability that a task is supervised miniimagenet')
 
 # os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.gpu)
 logdir = FLAGS.logdir
+print(FLAGS.p_gtgt)
 
 def train(model, saver, sess, exp_string, data_generator, resume_itr=0):
     SUMMARY_INTERVAL = 100
@@ -267,6 +269,8 @@ def main():
                 exp_string += '.scaled'
         if FLAGS.mt_mode == 'encenc':
             exp_string += '.ned_' + str(FLAGS.num_encoding_dims)
+        elif FLAGS.mt_mode == 'semi':
+            exp_string += '.pgtgt_' + str(FLAGS.p_gtgt)
     exp_string += '.mt_' + FLAGS.mt_mode
     exp_string += '.mbs_' + str(FLAGS.meta_batch_size) + \
                   '.nct_' + str(FLAGS.num_classes_train) + \

maml.py

@@ -40,15 +40,10 @@ def __init__(self, dim_input=1, dim_output_train=1, dim_output_val=1, test_num_u
                     if FLAGS.input_type == 'images_84x84':
                         self.forward = self.forward_resnet84
                         self.construct_weights = self.construct_resnet_weights84
+                        assert FLAGS.num_parts_per_res_block == 2
+                        assert FLAGS.num_res_blocks == 4
                         self.num_parts_per_res_block = FLAGS.num_parts_per_res_block
-                        blocks = ['input']
-                        for i in range(FLAGS.num_res_blocks):
-                            blocks.append('res{}'.format(i))
-                            if i != FLAGS.num_res_blocks - 1:
-                                blocks.append('maxpool')
-                        blocks.append('output')
-                        self.blocks = blocks
-                        print('blocks', self.blocks)
+                        self.blocks = ['input', 'maxpool', 'res0', 'maxpool', 'res1', 'maxpool', 'res2', 'maxpool', 'res3', 'output']
                     elif FLAGS.input_type == 'images_224x224':
                         self.forward = self.forward_resnet224
                         self.construct_weights = self.construct_resnet_weights224
@@ -338,9 +333,10 @@ def construct_resnet_weights84(self):
         conv_initializer = tf.contrib.layers.xavier_initializer_conv2d(dtype=dtype)
         bias_initializer = tf.zeros_initializer(dtype=dtype)
         fc_initializer = tf.contrib.layers.xavier_initializer(dtype=dtype)
-        def make_conv_layer_weights(weights, scope, k, filters_in, filters_out):
+        def make_conv_layer_weights(weights, scope, k, filters_in, filters_out, bias=True):
             weights['{}/conv'.format(scope)] = tf.get_variable('{}/conv'.format(scope), [k, k, filters_in, filters_out], initializer=conv_initializer, dtype=dtype)
-            weights['{}/bias'.format(scope)] = tf.get_variable('{}/bias'.format(scope), [filters_out], initializer=bias_initializer, dtype=dtype)
+            if bias:
+                weights['{}/bias'.format(scope)] = tf.get_variable('{}/bias'.format(scope), [filters_out], initializer=bias_initializer, dtype=dtype)
         def make_fc_layer_weights(weights, scope, dims_in, dims_out):
             weights['{}/fc'.format(scope)] = tf.get_variable('{}/fc'.format(scope), [dims_in, dims_out], initializer=fc_initializer, dtype=dtype)
             weights['{}/bias'.format(scope)] = tf.get_variable('{}/bias'.format(scope), [dims_out], initializer=bias_initializer, dtype=dtype)
@@ -349,11 +345,15 @@ def make_fc_layer_weights(weights, scope, dims_in, dims_out):
                 make_conv_layer_weights(weights, block_name, k=3, filters_in=self.channels, filters_out=64)
             elif 'res' in block_name:
                 j = int(block_name[-1])
-                last_block_filter = 64 if j == 0 else 64 * 2 ** (j-1)
+                last_block_filter = 64 if j == 0 else 64 * 2 ** (j - 1)
                 this_block_filter = 64 if j == 0 else last_block_filter * 2
                 print(block_name, last_block_filter, this_block_filter)
+                make_conv_layer_weights(weights, '{}/shortcut'.format(block_name), k=1, filters_in=last_block_filter,
+                                        filters_out=this_block_filter, bias=False)
                 for i in range(self.num_parts_per_res_block):
-                    make_conv_layer_weights(weights, '{}/part{}'.format(block_name, i), k=3, filters_in=64, filters_out=64)
+                    make_conv_layer_weights(weights, '{}/part{}'.format(block_name, i), k=3,
+                                            filters_in=last_block_filter if i == 0 else this_block_filter,
+                                            filters_out=this_block_filter)
             elif block_name == 'output':
                 make_fc_layer_weights(weights, block_name, dims_in=512, dims_out=self.dim_output_train)
         return weights
@@ -398,9 +398,10 @@ def forward_resnet84(self, inp, weights, prefix, reuse=False):
                 conv = weights['{}/conv'.format(block_name)]
                 bias = weights['{}/bias'.format(block_name)]
                 inp = tf.nn.conv2d(inp, filter=conv, strides=[1, 1, 1, 1], padding="SAME") + bias
-
             elif 'res' in block_name:
                 shortcut = inp
+                conv = weights['{}/shortcut/conv'.format(block_name)]
+                shortcut = tf.nn.conv2d(input=shortcut, filter=conv, strides=[1, 1, 1, 1], padding="SAME")
                 for part in range(self.num_parts_per_res_block):
                     part_name = 'part{}'.format(part)
                     scope = '{}/{}'.format(block_name, part_name)

scripts/miniimagenet_kmeans.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+num_clusters=500
+num_partitions=1    # 100 partitions takes a while
+num_encoding_dims=256
+encoder=deepcluster
+
+# training
+python3 main.py \
+    --dataset=miniimagenet --partition_algorithm=kmeans \
+    --save_checkpoints=False \
+    --num_classes_train=5 --num_classes_val=5 \
+    --inner_update_batch_size_train=1 --inner_update_batch_size_val=5 --outer_update_batch_size=5 \
+    --num_filters=32 --norm=batch_norm --max_pool=True --update_lr=0.05 --num_updates=5 \
+    --metatrain_iterations=60000 --meta_batch_size=8 --mt_mode=encenc --mv_mode=gtgt \
+    --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
+    --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
+    --logdir=./log/miniimagenet/kmeans-deepcluster \
+    --test_set=False --train=True \
+
+# testing
+for inner_update_batch_size_val in 1 5 20 50
+do
+    python3 main.py \
+        --dataset=miniimagenet --partition_algorithm=kmeans \
+        --save_checkpoints=False \
+        --num_classes_train=5 --num_classes_val=5 \
+        --inner_update_batch_size_train=1 --inner_update_batch_size_val=${inner_update_batch_size_val} --outer_update_batch_size=5 \
+        --num_filters=32 --norm=batch_norm --max_pool=True --update_lr=0.05 --num_updates=5 \
+        --metatrain_iterations=60000 --meta_batch_size=8 --mt_mode=encenc --mv_mode=gtgt \
+        --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
+        --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
+        --logdir=./log/miniimagenet/kmeans-deepcluster \
+        --test_set=True --train=False --log_inner_update_batch_size_val=5
+done
+

scripts/mnist_kmeans.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+num_clusters=10
+num_partitions=100
+num_encoding_dims=256
+encoder=acai
+
+python3 main.py \
+    --dataset=mnist --partition_algorithm=kmeans \
+    --save_checkpoints=False \
+    --num_classes_train=10 --num_classes_val=10 \
+    --inner_update_batch_size_train=1 --inner_update_batch_size_val=5 --outer_update_batch_size=10 \
+    --num_filters=32 --norm=batch_norm --max_pool=False --update_lr=0.05 --num_updates=5 \
+    --metatrain_iterations=20000 --meta_batch_size=16 --mt_mode=encenc --mv_mode=gtgt \
+    --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
+    --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
+    --logdir=./log/mnist/kmeans-acai \
+    --test_set=False --train=True
+for inner_update_batch_size_val in 1 5 10
+do
+    python3 main.py \
+        --dataset=mnist --partition_algorithm=kmeans \
+        --save_checkpoints=False \
+        --num_classes_train=10 --num_classes_val=10 \
+        --inner_update_batch_size_train=1 --inner_update_batch_size_val=${inner_update_batch_size_val} --outer_update_batch_size=10  \
+        --num_filters=32 --norm=batch_norm --max_pool=False --update_lr=0.05 --num_updates=5 \
+        --metatrain_iterations=20000 --meta_batch_size=16 --mt_mode=encenc --mv_mode=gtgt \
+        --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
+        --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
+        --logdir=./log/mnist/kmeans-acai \
+        --test_set=True --train=False --log_inner_update_batch_size_val=5
+done

scripts/omniglot_kmeans.sh

@@ -5,17 +5,17 @@ num_clusters=500
 num_partitions=1  # 100 partitions takes a while
 num_encoding_dims=256   # for bigan, change to num_encoding_dims=200; remember to change the logdir flag!
 encoder=acai    # for bigan, change to encoder=bigan
-#python3 main.py \
-#    --dataset=omniglot --partition_algorithm=kmeans --on_pixels=True \
-#    --save_checkpoints=False \
-#    --num_classes_train=20 --num_classes_val=5 \
-#    --inner_update_batch_size_train=1 --inner_update_batch_size_val=5 --outer_update_batch_size=5 \
-#    --num_filters=32 --norm=batch_norm --max_pool=False --update_lr=0.05 --num_updates=5 \
-#    --metatrain_iterations=30000 --meta_batch_size=8 --mt_mode=encenc --mv_mode=gtgt \
-#    --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
-#    --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
-#    --logdir=./log/omniglot/kmeans-pixels \
-#    --test_set=False --train=True
+python3 main.py \
+    --dataset=omniglot --partition_algorithm=kmeans --on_pixels=True \
+    --save_checkpoints=False \
+    --num_classes_train=20 --num_classes_val=5 \
+    --inner_update_batch_size_train=1 --inner_update_batch_size_val=5 --outer_update_batch_size=5 \
+    --num_filters=32 --norm=batch_norm --max_pool=False --update_lr=0.05 --num_updates=5 \
+    --metatrain_iterations=30000 --meta_batch_size=8 --mt_mode=encenc --mv_mode=gtgt \
+    --num_encoding_dims=${num_encoding_dims} --encoder=${encoder} \
+    --num_clusters=${num_clusters} --num_partitions=${num_partitions} \
+    --logdir=./log/omniglot/kmeans-pixels \
+    --test_set=False --train=True
 for inner_update_batch_size_val in 1 5
 do
     python3 main.py \