Skip to content

Commit

Permalink
set up
Browse files Browse the repository at this point in the history
  • Loading branch information
@LouisRanjard
LouisRanjard committed Dec 4, 2015
1 parent c101623 commit b1210be
Show file tree
Hide file tree
Showing 38 changed files with 2,465 additions and 0 deletions.
129 changes: 129 additions & 0 deletions GetHTKCoeffFromSong.m
View file
@@ -0,0 +1,129 @@
function [syllab] = GetHTKCoeffFromSong(song,filena,analysisdir,htkdir,filtering,normavec)
%
% song is the song datastructure to extract the syllables from
% filena is the syllab.mat datastructure file to save the syllab in
% analysisdir contains 1 or more configuration files for HCopy
% htkdir i sthe path to the HTK directory containing the HTK tools
% if filtering==1, then wavelet filtering is applied
% if normavec, then each coefficient (or group of coeff) is normalised so that it scales from 0 to 1
%
% example:
% load('/home/louis/Sounds/Tieke/K_01_2007/allpart/songs');
% [syllab] = GetHTKCoeffFromSong(song,'/home/louis/Sounds/Tieke/K_01_2007/allpart/syllab.mat','~/htk/config','/usr/local/bin/',);

if nargin<5, filtering=0; end
if numel(filtering)==0, filtering=0; end

syllab = [] ;

if isdir(fullfile(analysisdir))
conffiles = dir(fullfile(analysisdir,'*')) ;
else
conffiles = analysisdir;
end

for sg=1:numel(song)
%[status, filename] = system(['locate -n1 -r ?*/' song(sg).filename '$']);
filename = which(song(sg).filename);
if numel(strfind(filename,song(sg).filename)==1)
%fprintf('%s\n',song(sg).filename);
%filename=filename(1:end-1) ; % remove the \n at the end
[ y ] = wavread(filename);
y = reshape(y,1,[]) ; % make sure the first dimension is 1
% DeNoise the whole recording
% QMF = MakeONFilter('Daubechies',8);
% scaledsong = NormNoise(y,QMF);
% % find the difference between the song length and the closest factor of 2 number
% dif2 = 2^(ceil(log2(length(scaledsong))))-length(scaledsong) ;
% scaledsong = [ scaledsong zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
% [sylsig,wcoef] = WaveShrink(scaledsong,'Visu',5,QMF);
% y = sylsig(1:end-dif2) ;
m = 1 ; % index of syllables for this particular song
for k = (length(syllab)+1) : (length(syllab)+length(song(sg).SyllableS))
seqvect = [] ;
sylfilename0 = fullfile('/tmp','/syltmp.wav') ;
sylfilename1 = fullfile('/tmp','/syltmp.coeff') ;
sylsig = y(floor(song(sg).SyllableS(m)):floor(song(sg).SyllableE(m))) ;
% FILTERING
if filtering>0
% Wavelet DeNoising
% USING WaveShrink
% QMF = MakeONFilter('Symmlet',8);
% QMF = MakeONFilter('Daubechies',20);
% QMF = MakeONFilter('Haar',8);
% QMF = MakeONFilter('Daubechies',8);
% scaledsong = NormNoise(sylsig,QMF);
% % find the difference between the song length and the closest factor of 2 number
% dif2 = 2^(ceil(log2(length(scaledsong))))-length(scaledsong) ;
% scaledsong = [ scaledsong zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
% [sylsig,wcoef] = WaveShrink(scaledsong,'Visu',5,QMF);
% sylsig = sylsig(1:end-dif2) ;
% USING CohWave
% longueur = length(sylsig) ;
% sylsig = CohWave([reshape(sylsig,1,[]) zeros(1,2^ceil(log2(length(sylsig)))-length(sylsig))],5,MakeONFilter('Beylkin'));
% sylsig = sylsig(1:longueur) ;
% USING WPDeNoise coiflet
% dif2 = 2^(ceil(log2(length(sylsig))))-length(sylsig) ;
% scaledsong = [ sylsig zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
% D = 5;
% QCoif8 = MakeONFilter('Coiflet',8);
% sylsig = WPDeNoise(scaledsong,D,QCoif8);
% USING WPDeNoise daubechies
% QMF = MakeONFilter('Daubechies',8);
% sylsig = NormNoise(sylsig,QMF);
% dif2 = 2^(ceil(log2(length(sylsig))))-length(sylsig) ;
% scaledsong = [ sylsig zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
% D = 5;
% sylsig = WPDeNoise(scaledsong,D,QMF);
%sylsig = CPDeNoise(scaledsong,D,'Sine');
% USING Short Course 28: Robust De-Noising
% dif2 = 2^(ceil(log2(length(sylsig))))-length(sylsig) ;
% scaledsong = [ sylsig zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
% wc = FHT_Med(scaledsong);
% wc(triad(6)) = 0 .* wc(triad(6)) ;
% sylsig = IHT_Med(wc);
% USING wavelet thresholding
dif2 = 2^(ceil(log2(length(sylsig))))-length(sylsig) ;
scaledsong = [ sylsig zeros(1,dif2) ] ; % watch out scaledsong length must be a power of 2
sylsig = ThreshWave(scaledsong) ;
end
wavwrite(sylsig,44100,16,sylfilename0) ;
% ENCODE WITH HTK CONFIG FILES
for cff=1:numel(conffiles)
if isdir(fullfile(analysisdir,conffiles(cff).name)), continue; end % avoid directories (. ..)
% USING HTK TO GET THE COEFF
%system([fullfile(htkdir,'HCopy') ' -A -C ' fullfile(analysisdir,conffiles(cff).name) ' ' sylfilename0 ' ' sylfilename1]) ;
system([fullfile(htkdir,'HCopy') ' -C ' fullfile(analysisdir,conffiles(cff).name) ' ' sylfilename0 ' ' sylfilename1]) ; % same but suppress output
[coeffseq,fp] = readhtk(sylfilename1) ;
seqvect = [seqvect coeffseq]; % matrix transposed with readhtk (compared to readmfcc)
delete(sylfilename1) ;
end
% NORMALISE EACH COEFF
if nargin>5
% normalise from 0 to 1 according to a vector giving the structure, e.g. [1 12; 13 13; 14 25]
seqvect = norma_seqvect(seqvect',normavec) ; % NEED TO TRANSPOSE THE MATRIX
end
syllab(k).seqvect = seqvect' ;
delete(sylfilename0) ;
m = m+1 ;
end
else
fprintf('file not found %s\n',song(sg).filename);
end
% save( [filena '.tmp'] , 'syllab' ) ;
end

% check if it is required to transpose the matrices, DTWaverage needs the time in the second dimension
tmp = struct2cell(syllab) ;
sizdim1 = cellfun(@(x) size(x,1),tmp) ;
if var(sizdim1)>0 % transpose every matrices
tmp = cellfun(@(x) x',tmp,'UniformOutput',false) ;
end
syllab = cell2struct(tmp,'seqvect') ;

% normalization (no need, HTK performs E normalisation)
%syllab = cell2struct( cellfun(@(x) norma(x,0,1),{syllab.htkmfcc}, 'UniformOutput', false) , 'htkmfcc' , 1 )' ;
if numel(filena)>0
save( '-v7', filena , 'syllab' ) ;
end

97 changes: 97 additions & 0 deletions compare_TextGrid.m
View file
@@ -0,0 +1,97 @@
function [similarity] = compare_TextGrid(filename1,filename2,binsiz,N)
% compare two TextGrid files
% need to have the wav files in the same directory (to get the Fs)
% if N provided build a null distribution of similarity using N random countlab matrices
% binsiz: size of the bins to compare the two textgrid files in seconds
% example: [similarity] = compare_TextGrid('/Sounds/Todd_files/Todd_Meeting_23JAN10/manual_annotation/end_of_01_manual_annotation.TextGrid','/Sounds/Todd_files/Todd_Meeting_23JAN10/test_quackass_window100ms/end_of_01_manual_annotation.TextGrid');

if nargin<3
% choose bin size in seconds
binsiz = 30 ;
end

% get the Fs
tmp1=regexprep(filename1(end:-1:1),'dirGtxeT.','vaw.','once');tmp1=tmp1(end:-1:1); % allows to replace just once, the last one
[y1, Fs1] = wavread(tmp1,1) ;
length1 = wavread(tmp1,'size') ;
tmp2=regexprep(filename2(end:-1:1),'dirGtxeT.','vaw.','once');tmp2=tmp2(end:-1:1); % allows to replace just once, the last one
[y2, Fs2] = wavread(tmp2,1) ;
length2 = wavread(tmp2,'size') ;
if Fs1~=Fs2
error('sampling frequencies are different');
else
Fs=Fs1;
end

% create song structures
tmp1=regexprep(filename1(end:-1:1),'dirGtxeT.','flm.','once');tmp1=tmp1(end:-1:1); % allows to replace just once, the last one
textGrid2mlf( filename1, tmp1 ) ;
song1 = mlf2song( tmp1, [], 3) ;

tmp2=regexprep(filename2(end:-1:1),'dirGtxeT.','flm.','once');tmp2=tmp2(end:-1:1); % allows to replace just once, the last one
textGrid2mlf( filename2, tmp2 ) ;
song2 = mlf2song( tmp2, [], 3) ;

% check for compatibility between the songs, need exactly the same syllable types, if not add zero length syllables at the end
for missing = setdiff( unique(song2.sequence),unique(song1.sequence) )
song1.sequence = [song1.sequence missing] ;
song1.sequencetxt = [song1.sequencetxt song2.sequencetxt(find(song2.sequence==missing,1,'first')) ] ;
song1.SyllableS = [song1.SyllableS song1.SyllableE(end)] ;
song1.SyllableE = [song1.SyllableE song1.SyllableE(end)] ;
end
for missing = setdiff( unique(song1.sequence),unique(song2.sequence) )
song2.sequence = [song2.sequence missing] ;
song2.sequencetxt = [song2.sequencetxt song1.sequencetxt(find(song1.sequence==missing,1,'first')) ] ;
song2.SyllableS = [song2.SyllableS song2.SyllableE(end)] ;
song2.SyllableE = [song2.SyllableE song2.SyllableE(end)] ;
end

% create tables to use the comparison process
syltable1 = song2table(song1) ;
countlab1 = syltable_bins(syltable1,binsiz,Fs,length1(1)) ;

syltable2 = song2table(song2) ;
countlab2 = syltable_bins(syltable2,binsiz,Fs,length2(1)) ;

% check conversion has been okay (allows 0.01 error)
if sum( (sum(countlab1,2)>binsiz*1.01 | sum(countlab1,2)<binsiz*0.99)>0 )
error('conversion problem on file 1');
end
if sum( (sum(countlab2,2)>binsiz*1.01 | sum(countlab2,2)<binsiz*0.99)>0 )
error('conversion problem on file 2');
end

similarity = simcountlab(countlab1,countlab2,binsiz) ;
fprintf(1,'similarity = %.2f%%\n',similarity*100) ;

% do randomization test?
if nargin>3
nullsim = zeros(1,N) ;
for permut=1:N
% randomly shuffle the second matrix
cntlabrand =zeros(size(countlab2,1),size(countlab2,2)) ;
for size1=1:size(countlab2,1)
cntlabrand(size1,:) = countlab2(size1,randperm(size(countlab2,2))) ;
end
% randomly permute the rows and columns of the second matrix
% cntlabrand = countlab2(randperm(size(countlab2,1)),randperm(size(countlab2,2))) ;
% randomly fill up a new matrix
% cntlabrand = rand(size(countlab2,1),size(countlab2,2)) ;
% cntlabrand = cntlabrand ./ repmat(sum(cntlabrand,2),1,size(countlab2,2)) ;
% cntlabrand = cntlabrand.*binsiz ;
% compute the similarity score with the random matrix
nullsim(permut) = simcountlab(countlab1,cntlabrand,binsiz) ;
end
% pvalue computated with alpha=5%
fprintf(1,'p-value = %.4f (%.0f random permutations, mean=%.2f%%)\n',sum(nullsim>=similarity)/N,N,mean(nullsim)*100) ;
end

function [simscore] = simcountlab(cntlab1,cntlab2,binsiz)
% euclidean distance of each row vector
%eucldist = sqrt( sum( (cntlab1(:,2:end)-cntlab2(:,2:end)).^2 ,2) ) ;
eucldist = ( sum( (cntlab1(:,2:end)-cntlab2(:,2:end)).^2 ,2) ) .^ (1/2) ;
% divide each euclidean distance by the maximum distance possible which is the size of the bin times two
simscore = 1 - (sum(eucldist)/numel(eucldist))/(binsiz*2) ;
end

end
97 changes: 97 additions & 0 deletions compare_label.m
View file
@@ -0,0 +1,97 @@
function [similarity] = compare_label(filename1,filename2,binsiz,N)
% compare two LABEL files
% need to have the wav files in the same directory (to get the Fs)
% if N provided build a null distribution of similarity using N random countlab matrices
% binsiz: size of the bins to compare the two textgrid files in seconds
% example: [similarity] = compare_label('/Sounds/Todd_files/Todd_Meeting_23JAN10/manual_annotation/end_of_01_manual_annotation.label','/Sounds/Todd_files/Todd_Meeting_23JAN10/test_quackass_window100ms/end_of_01_manual_annotation.label');

if nargin<3
% choose bin size in seconds
binsiz = 30 ;
end

% get the Fs
tmp1=regexprep(filename1(end:-1:1),'lebal.','vaw.','once');tmp1=tmp1(end:-1:1); % allows to replace just once, the last one
[~, Fs1] = wavread(tmp1,1) ;
length1 = wavread(tmp1,'size') ;
tmp2=regexprep(filename2(end:-1:1),'lebal.','vaw.','once');tmp2=tmp2(end:-1:1); % allows to replace just once, the last one
[~, Fs2] = wavread(tmp2,1) ;
length2 = wavread(tmp2,'size') ;
if Fs1~=Fs2
error('sampling frequencies are different');
else
Fs=Fs1;
end

% create song structures
tmp1=regexprep(filename1(end:-1:1),'lebal.','flm.','once');tmp1=tmp1(end:-1:1); % allows to replace just once, the last one
label2mlf( filename1, tmp1 ) ;
song1 = mlf2song( tmp1, [], 3) ;

tmp2=regexprep(filename2(end:-1:1),'lebal.','flm.','once');tmp2=tmp2(end:-1:1); % allows to replace just once, the last one
label2mlf( filename2, tmp2 ) ;
song2 = mlf2song( tmp2, [], 3) ;

% check for compatibility between the songs, need exactly the same syllable types, if not add zero length syllables at the end
for missing = setdiff( unique(song2.sequence),unique(song1.sequence) )
song1.sequence = [song1.sequence missing] ;
song1.sequencetxt = [song1.sequencetxt song2.sequencetxt(find(song2.sequence==missing,1,'first')) ] ;
song1.SyllableS = [song1.SyllableS song1.SyllableE(end)] ;
song1.SyllableE = [song1.SyllableE song1.SyllableE(end)] ;
end
for missing = setdiff( unique(song1.sequence),unique(song2.sequence) )
song2.sequence = [song2.sequence missing] ;
song2.sequencetxt = [song2.sequencetxt song1.sequencetxt(find(song1.sequence==missing,1,'first')) ] ;
song2.SyllableS = [song2.SyllableS song2.SyllableE(end)] ;
song2.SyllableE = [song2.SyllableE song2.SyllableE(end)] ;
end

% create tables to use the comparison process
syltable1 = song2table(song1) ;
countlab1 = syltable_bins(syltable1,binsiz,Fs,length1(1)) ;

syltable2 = song2table(song2) ;
countlab2 = syltable_bins(syltable2,binsiz,Fs,length2(1)) ;

% check conversion has been okay (allows 0.01 error)
if sum( (sum(countlab1,2)>binsiz*1.01 | sum(countlab1,2)<binsiz*0.99)>0 )
error('conversion problem on file 1');
end
if sum( (sum(countlab2,2)>binsiz*1.01 | sum(countlab2,2)<binsiz*0.99)>0 )
error('conversion problem on file 2');
end

similarity = simcountlab(countlab1,countlab2,binsiz) ;
fprintf(1,'similarity = %.2f%%\n',similarity*100) ;

% do randomization test?
if nargin>3
nullsim = zeros(1,N) ;
for permut=1:N
% randomly shuffle the second matrix
cntlabrand =zeros(size(countlab2,1),size(countlab2,2)) ;
for size1=1:size(countlab2,1)
cntlabrand(size1,:) = countlab2(size1,randperm(size(countlab2,2))) ;
end
% randomly permute the rows and columns of the second matrix
% cntlabrand = countlab2(randperm(size(countlab2,1)),randperm(size(countlab2,2))) ;
% randomly fill up a new matrix
% cntlabrand = rand(size(countlab2,1),size(countlab2,2)) ;
% cntlabrand = cntlabrand ./ repmat(sum(cntlabrand,2),1,size(countlab2,2)) ;
% cntlabrand = cntlabrand.*binsiz ;
% compute the similarity score with the random matrix
nullsim(permut) = simcountlab(countlab1,cntlabrand,binsiz) ;
end
% pvalue computated with alpha=5%
fprintf(1,'p-value = %.4f (%.0f random permutations, mean=%.2f%%)\n',sum(nullsim>=similarity)/N,N,mean(nullsim)*100) ;
end

function [simscore] = simcountlab(cntlab1,cntlab2,binsiz)
% euclidean distance of each row vector
%eucldist = sqrt( sum( (cntlab1(:,2:end)-cntlab2(:,2:end)).^2 ,2) ) ;
eucldist = ( sum( (cntlab1(:,2:end)-cntlab2(:,2:end)).^2 ,2) ) .^ (1/2) ;
% divide each euclidean distance by the maximum distance possible which is the size of the bin times two
simscore = 1 - (sum(eucldist)/numel(eucldist))/(binsiz*2) ;
end

end

0 comments on commit b1210be

Please sign in to comment.