From 041c6c08d58a4af175d74a69d28c5c500f330840 Mon Sep 17 00:00:00 2001 From: Yu-Wei Chao Date: Wed, 17 Jun 2015 13:50:22 -0400 Subject: [PATCH] visualize 2d affordance space --- README.md | 20 ++-- addPaths.m | 1 + common/pca_shlens_1.m | 30 +++++ config.m | 1 + visualize/get_mscoco_cat_info.m | 111 ++++++++++++++++++ visualize/pca_2d_run.m | 199 ++++++++++++++++++++++++++++++++ 6 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 common/pca_shlens_1.m create mode 100644 visualize/get_mscoco_cat_info.m create mode 100644 visualize/pca_2d_run.m diff --git a/README.md b/README.md index facf649..162e0d2 100644 --- a/README.md +++ b/README.md @@ -19,19 +19,23 @@ Check out the [project site](http://www.umich.edu/~ywchao/semantic_affordance/) ## Quick start -1. Download a copy of our [affordance dataset](http://www.umich.edu/~ywchao/semantic_affordance/data/affordance_data.tar.gz) and unzip the file. +0. Download a copy of our [affordance dataset](http://www.umich.edu/~ywchao/semantic_affordance/data/affordance_data.tar.gz) and unzip the file. -2. Change Matlab's current directory into the directory of this README. +0. Get the source code by cloning the repository: `git clone https://github.com/ywchao/semantic_affordance.git` -3. Change the path `data_dir` in `config.m` to the downloaded folder `affordance_data/`. -```MATLAB -data_dir = '/z/ywchao/datasets/affordance_data/'; -``` +0. Change into the source code directory `cd semantic_affordance` and start MATLAB `matlab`. You should see the message `added paths for the experiment!` followed by the MATLAB prompt `>>`. -4. Run `setup` to prepare the required files. +0. Change the path `data_dir` in `config.m` to the downloaded folder `affordance_data/`. + 
+  data_dir = '/z/ywchao/datasets/affordance_data/';
+
+ +0. Run `setup` to prepare the required files. - Generate ground-truth binary labels from afforadance data - Generate WordNet similarity measures for 91 MS-COCO object categories - Download KPMF code -5. Run `demo_cf_nn` and `demo_cf_kpmf` to reproduce the NN and KPMF results. +0. Run `pca_2d_run` to visualize the object categories in the 2D affordance space. + +0. Run `demo_cf_nn` and `demo_cf_kpmf` to reproduce the NN and KPMF results. diff --git a/addPaths.m b/addPaths.m index 0d944c1..f6b292c 100644 --- a/addPaths.m +++ b/addPaths.m @@ -12,3 +12,4 @@ addpath('vn_predict/'); addpath('wn_similarity/'); addpath('utils/'); +addpath('visualize/'); diff --git a/common/pca_shlens_1.m b/common/pca_shlens_1.m new file mode 100644 index 0000000..cd3dc8c --- /dev/null +++ b/common/pca_shlens_1.m @@ -0,0 +1,30 @@ +function [ signals, PC, V ] = pca_shlens_1( data ) +% PCA1: Perform PCA using covariance. +% data - MxN matrix of input data +% (M dimensions, N trials) +% signals - MxN matrix of projected data +% PC - each column is a PC +% V - Mx1 matrix of variances + +[M,N] = size(data); %#ok + +% subtract off the mean for each dimension +mn = mean(data,2); +data = data - repmat(mn,1,N); + +% calculate the covariance matrix +covariance = 1 / (N-1) * (data * data'); + +% find the eigenvectors and eigenvalues +[PC, V] = eig(covariance); + +% extract diagonal of matrix as vector +V = diag(V); + +% sort the variances in decreasing order +[junk, rindices] = sort(-1*V); %#ok +V = V(rindices); +PC = PC(:,rindices); + +% project the original data set +signals = PC' * data; diff --git a/config.m b/config.m index aebbe20..b68c822 100644 --- a/config.m +++ b/config.m @@ -9,6 +9,7 @@ wnsim_base = fullfile(base_dir,'cache/wn_similarity/'); cf_base = fullfile(base_dir,'cache/collab_filt/'); eval_base = fullfile(base_dir,'cache/evaluation/'); +vis_base = fullfile(base_dir,'cache/visualize/'); % wordnet similarities n_list_file = [wnsim_base 'n_mscoco_list']; diff --git a/visualize/get_mscoco_cat_info.m b/visualize/get_mscoco_cat_info.m new file mode 100644 index 0000000..16cd6cc --- /dev/null +++ b/visualize/get_mscoco_cat_info.m @@ -0,0 +1,111 @@ +function [ cat_name, cat_ind, cat_color ] = get_mscoco_cat_info( ) + +% cat_name +cat_name{1} = 'person & Accessory'; +cat_name{2} = 'Animal'; +cat_name{3} = 'Vehicle'; +cat_name{4} = 'Outdoor Objects'; +cat_name{5} = 'Sports'; +cat_name{6} = 'Kitchenware'; +cat_name{7} = 'Food'; +cat_name{8} = 'Furniture'; +cat_name{9} = 'Appliance'; +cat_name{10} = 'Electronics'; +cat_name{11} = 'Indoor Objects'; + +cat_color = distinguishable_colors(15); + +cat_ind(1) = 1; % person +cat_ind(2) = 3; % bicycle +cat_ind(3) = 3; % car +cat_ind(4) = 3; % motorcycle +cat_ind(5) = 3; % airplace +cat_ind(6) = 3; % bus +cat_ind(7) = 3; % train +cat_ind(8) = 3; % truck +cat_ind(9) = 3; % boat +cat_ind(10) = 4; % traffic_light +cat_ind(11) = 4; % fire_hydrant +cat_ind(12) = 4; % street_sign +cat_ind(13) = 4; % stop_sign +cat_ind(14) = 4; % parking_meter +cat_ind(15) = 4; % bench +cat_ind(16) = 2; % bird +cat_ind(17) = 2; % cat +cat_ind(18) = 2; % dog +cat_ind(19) = 2; % horse +cat_ind(20) = 2; % sheep +cat_ind(21) = 2; % cow +cat_ind(22) = 2; % elephant +cat_ind(23) = 2; % bear +cat_ind(24) = 2; % zebra +cat_ind(25) = 2; % giraffe +cat_ind(26) = 1; % hat +cat_ind(27) = 1; % backpack +cat_ind(28) = 1; % umbrella +cat_ind(29) = 1; % shoe +cat_ind(30) = 1; % eye_glasses +cat_ind(31) = 1; % handbag +cat_ind(32) = 1; % tie +cat_ind(33) = 1; % suitcase +cat_ind(34) = 5; % frisbee +cat_ind(35) = 5; % skis +cat_ind(36) = 5; % snowboard +cat_ind(37) = 5; % sports_ball +cat_ind(38) = 5; % kite +cat_ind(39) = 5; % baseball_bat +cat_ind(40) = 5; % baseball_glove +cat_ind(41) = 5; % skateboard +cat_ind(42) = 5; % surfboard +cat_ind(43) = 5; % tennis_racket +cat_ind(44) = 6; % bottle +cat_ind(45) = 6; % plate +cat_ind(46) = 6; % wine_glass +cat_ind(47) = 6; % cup +cat_ind(48) = 6; % fork +cat_ind(49) = 6; % knife +cat_ind(50) = 6; % spoon +cat_ind(51) = 6; % bowl +cat_ind(52) = 7; % banana +cat_ind(53) = 7; % apple +cat_ind(54) = 7; % sandwich +cat_ind(55) = 7; % orange +cat_ind(56) = 7; % broccoli +cat_ind(57) = 7; % carrot +cat_ind(58) = 7; % hot_dog +cat_ind(59) = 7; % pizza +cat_ind(60) = 7; % donut +cat_ind(61) = 7; % cake +cat_ind(62) = 8; % chair +cat_ind(63) = 8; % couch +cat_ind(64) = 8; % potted plant +cat_ind(65) = 8; % bed +cat_ind(66) = 8; % mirror +cat_ind(67) = 8; % dining_table +cat_ind(68) = 8; % window +cat_ind(69) = 8; % desk +cat_ind(70) = 8; % toilet +cat_ind(71) = 8; % door +cat_ind(72) = 10; % tv +cat_ind(73) = 10; % laptop +cat_ind(74) = 10; % mouse +cat_ind(75) = 10; % remote +cat_ind(76) = 10; % keyboard +cat_ind(77) = 10; % cell_phone +cat_ind(78) = 9; % microwave +cat_ind(79) = 9; % oven +cat_ind(80) = 9; % toaster +cat_ind(81) = 9; % sink +cat_ind(82) = 9; % refrigerator +cat_ind(83) = 9; % blender +cat_ind(84) = 11; % book +cat_ind(85) = 11; % clock +cat_ind(86) = 11; % vase +cat_ind(87) = 11; % scissors +cat_ind(88) = 11; % teddy_bear +cat_ind(89) = 11; % hair_drier +cat_ind(90) = 11; % toothbrush +cat_ind(91) = 11; % hair_brush + +end + diff --git a/visualize/pca_2d_run.m b/visualize/pca_2d_run.m new file mode 100644 index 0000000..1913474 --- /dev/null +++ b/visualize/pca_2d_run.m @@ -0,0 +1,199 @@ +% visualize objects in the 2D affordance space + +config; + +% load mcoco_list +load(mcoco_file); + +% set parameters +num_verb = 957; +N = 30; % number of significant verbs to visualize + +% make directories +makedir(vis_base); + +%% visualize the affordance space of 20 pascal object categories +% (figure 4 in the paper) +fprintf('visualizing 20 pascal object categories ... \n'); +fprintf(' output to %s\n',vis_base); + +% get ids & nnames +ID = [mcoco_list.id]'; +NNAME = {mcoco_list.name}'; +NNAME = cellfun(@(x)strrep(x,' ','_'),NNAME,'UniformOutput',false); + +% keep only the pascal objects +ID = ID([mcoco_list.is_pascal] == 1); +NNAME = NNAME([mcoco_list.is_pascal] == 1); +num_noun = numel(NNAME); + +% sort by nname +[NNAME,ii] = sort(NNAME); +ID = ID(ii); + +% load scores +X = zeros(num_verb, num_noun); +for n = 1:num_noun + nname = NNAME{n}; + data_file = fullfile(data_dir,sprintf('%02d_%s.mat',ID(n),NNAME{n})); + load(data_file,'data'); + + x = [data.score] - 3; + X(:,n) = x'; +end + +% run pca +[signals,PC,V] = pca_shlens_1(X); + +% visualize objects on 2D +pt_2d = signals(1:2,:); + +figure(1); clf; +for n = 1:num_noun + plot(pt_2d(1,n),pt_2d(2,n),'ro','MarkerFaceColor','r'); hold on + text(pt_2d(1,n)+0.5,pt_2d(2,n)+0, ... + strrep(NNAME{n},'_',' '), ... + 'FontSize',12); +end +grid on + +fig_name = [vis_base,'pca_pascal.pdf']; +if ~exist(fig_name,'file') + print(gcf,'-dpdf',fig_name); +end +close; + +% visualize responses of the significant verbs +p = max(abs([PC(:,1) PC(:,2)]),[],2); +[~,ii] = sort(abs(p),'descend'); + +ii_N = ii(1:N); +verb_syn = [data.verb_syn]; +SIG_VERB = {verb_syn(ii_N).name}'; +SIG_VERB = cellfun(@(x)strrep(x,'_',' '),SIG_VERB,'UniformOutput',false); + +sv_dir = [vis_base sprintf('sig_verb_%03d_pascal',N) '/']; +makedir(sv_dir); +figure(1); +for n = 1:num_noun + clf; + barh(X(ii_N,n)'+3); + set(gca,'YDir','Reverse') + axis([0 5 0.5 N+0.5]); + set(gca,'YTick',1:1:N) + set(gca,'XTick',0:1:5) + set(gca,'YTickLabel',SIG_VERB); + set(gca, 'Ticklength', [0 0]) + grid on; + ht = title(sprintf('%s',strrep(NNAME{n},'_',' '))); + + set(gcf,'PaperPositionMode','manual'); + set(ht,'FontSize',20); + set(gca,'FontSize',12); + set(gcf,'PaperPosition',[0.25 0.25 3.1 6]); + + fig_name = [sv_dir,sprintf('%s.pdf',NNAME{n})]; + if ~exist(fig_name,'file') + print(gcf,'-dpdf',fig_name); + end +end +close; +fprintf('done.\n'); + +%% visualize the affordance space of 91 mscoco object categories +fprintf('visualizing 91 mscoco object categories ... \n'); +fprintf(' output to %s\n',vis_base); + +% get ids & nnames +ID = [mcoco_list.id]'; +NNAME = {mcoco_list.name}'; +NNAME = cellfun(@(x)strrep(x,' ','_'),NNAME,'UniformOutput',false); +num_noun = numel(NNAME); + +% sort by nname +[NNAME,ii] = sort(NNAME); +ID = ID(ii); + +% load scores +X = zeros(num_verb, num_noun); +for n = 1:num_noun + nname = NNAME{n}; + data_file = fullfile(data_dir,sprintf('%02d_%s.mat',ID(n),NNAME{n})); + load(data_file,'data'); + + x = [data.score] - 3; + X(:,n) = x'; +end + +% run pca +[signals,PC,V] = pca_shlens_1(X); + +% load mscoco cat info +[cat_name, cat_ind, cat_color] = get_mscoco_cat_info(); +cat_ind = cat_ind(ii); + +% visualize objects on 2D +pt_2d = signals(1:2,:); + +figure(1); clf; +hp = zeros(num_noun,1); +for n = 1:num_noun + hp(n) = plot( ... + pt_2d(1,n),pt_2d(2,n),'o', ... + 'Color',cat_color(cat_ind(n),:), ... + 'MarkerSize',3, ... + 'MarkerFaceColor',cat_color(cat_ind(n),:) ... + ); + hold on + text(pt_2d(1,n)+0.5,pt_2d(2,n)+0, ... + strrep(NNAME{n},'_',' '), ... + 'FontSize',4); +end +grid on +legend( ... + hp([3 7 1 9 5 15 2 8 12 23 14]),cat_name, ... + 'Location','southeast' ... + ); + +fig_name = [vis_base,'pca_mscoco.pdf']; +if ~exist(fig_name,'file') + print(gcf,'-dpdf',fig_name); +end +close; + +% visualize responses of the significant verbs +p = max(abs([PC(:,1) PC(:,2)]),[],2); +[~,ii] = sort(abs(p),'descend'); + +ii_N = ii(1:N); +verb_syn = [data.verb_syn]; +SIG_VERB = {verb_syn(ii_N).name}'; +SIG_VERB = cellfun(@(x)strrep(x,'_',' '),SIG_VERB,'UniformOutput',false); + +sv_dir = [vis_base sprintf('sig_verb_%03d_mscoco',N) '/']; +makedir(sv_dir); +figure(1); +for n = 1:num_noun + clf; barh(X(ii_N,n)'+3); + set(gca,'YDir','Reverse') + axis([0 5 0.5 N+0.5]); + set(gca,'YTick',1:1:N) + set(gca,'XTick',0:1:5) + set(gca,'YTickLabel',SIG_VERB); + set(gca, 'Ticklength', [0 0]) + grid on; + ht = title(sprintf('%s',strrep(NNAME{n},'_',' '))); + + set(gcf,'PaperPositionMode','manual'); + set(ht,'FontSize',20); + set(gca,'FontSize',12); + set(gcf,'PaperPosition',[0.25 0.25 3.1 6]); + + fig_name = [sv_dir,sprintf('%s.pdf',NNAME{n})]; + if ~exist(fig_name,'file') + print(gcf,'-dpdf',fig_name); + end +end +close; +fprintf('done.\n'); +