Skip to content

Commit

Permalink
ampact01
Browse files Browse the repository at this point in the history
  • Loading branch information
@jcdevaney
jcdevaney committed Oct 8, 2011
1 parent 2027d4e commit ac6cbab
Show file tree
Hide file tree
Showing 25 changed files with 1,367 additions and 0 deletions.
Binary file added SingingMeansCovars.mat
View file
Binary file not shown.
115 changes: 115 additions & 0 deletions alignmentVisualiser.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
function alignmentVisualiser(trace,mid,spec,fig)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% alignmentVisualiser(trace,sig,sr,mid,highlight)
%
% Description:
% Plots a gross DTW alignment overlaid with the fine alignment
% resulting from the HMM aligner on the output of YIN. Trace(1,:)
% is the list of states in the HMM, and trace(2,:) is the number of YIN
% frames for which that state is occupied. Highlight is a list of
% notes for which the steady state will be highlighted.
%
% Inputs:
% trace - 3-D matrix of a list of states (trace(1,:)), the times
% they end at (trace(2,:)), and the state indices (trace(3,:))
% mid - midi file
% spec - spectogram of audio file (from alignmidiwav.m)
%
% Dependencies:
% Toiviainen, P. and T. Eerola. 2006. MIDI Toolbox. Available from:
% https://www.jyu.fi/hum/laitokset/musiikki/en/research/coe/materials
% /miditoolbox/
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca) and Michael Mandel
% (mim@mr-pc.org), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

if ~exist('fig', 'var'), fig=1; end

% Fix for ending zeros that mess up the plot
if trace(2,end)==0
trace=trace(:,1:end-1);
end
if trace(2, end-1)==0
trace(2,end-1)=trace(2,end-2);
end

% hop size between frames
stftHop = 0.025;

% read midi file
nmat=readmidi(mid);

% plot spectogram of audio file
figure(fig)
imagesc(20*log10(spec));
title(['Spectrogram with Aligned MIDI Notes Overlaid']);
xlabel(['Time (.05s)']);
ylabel(['Midinote']);
axis xy;
caxis(max(caxis)+[-50 0])
colormap(1-gray)

% zoom in fundamental frequencies
notes = nmat(:,4)';
notes = (2.^((notes-105)/12))*440;
notes(end+1) = notes(end);
nlim = length(notes);

% plot alignment
plotFineAlign(trace(1,:), trace(2,:), notes(1:nlim), stftHop);
if size(trace,1) >= 3
notenums = trace(3,2:end);
else
nlim = length(notes);
notenums = [reshape(repmat(1:nlim,4,1),1,[]) nlim];
end


function plotFineAlign(stateType, occupancy, notes, stftHop)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% plotFineAlign(stateType, occupancy, notes, stftHop, highlight)
%
% Description:
% Plot the HMM alignment based on the output of YIN. StateType is the
% list of states in the HMM, and occupancy is the number of YIN frames
% for which that state is occupied. Notes is a list of midi note numbers
% that are played, should be one note for each [3] in stateType. If the
% highlight vector is supplied, it should contain indices of the states
% to highlight by plotting an extra line at the bottom of the window.
%
% Inputs:
% stateType - vector with a list of states
% occupancy - vector indicating the time (in seconds) at which the states
% in stateType end
% notes - vector of notes from MIDI file
% stftHop - the hop size between frames in the spectrogram
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca) and Michael Mandel
% (mim@mr-pc.org), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Plot the 4 states: silence in red, beginning transient in green,
% steady state in blue, ending transient in green.

styles = {{'r+-', 'LineWidth', 2},
{'g+-', 'LineWidth', 2},
{'b+-', 'LineWidth', 2}};

cs = occupancy /stftHop;
segments = [cs(1:end-1); cs(2:end)]';

hold on

stateNote = max(1, cumsum(stateType == 3)+1);
for i=1:size(segments,1)
plot(segments(i,:)', repmat(notes(stateNote(i)),2,1), styles{stateType(i+1)}{:})
end

hold off
112 changes: 112 additions & 0 deletions alignmentVisualiser.m~
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
function alignmentVisualiser(trace,mid,spec,fig)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% alignmentVisualiser(trace,sig,sr,mid,highlight)
%
% Description:
% Plots a gross DTW alignment overlaid with the fine alignment
% resulting from the HMM aligner on the output of YIN. Trace(1,:)
% is the list of states in the HMM, and trace(2,:) is the number of YIN
% frames for which that state is occupied. Highlight is a list of
% notes for which the steady state will be highlighted.
%
% Inputs:
% trace - 3-D matrix of a list of states (trace(1,:)), the times
% they end at (trace(2,:)), and the state indices (trace(3,:))
% mid - midi file
% spec - spectogram of audio file (from alignmidiwav.m)
%
% Dependencies:
% Toiviainen, P. and T. Eerola. 2006. MIDI Toolbox. Available from:
% https://www.jyu.fi/hum/laitokset/musiikki/en/research/coe/materials
% /miditoolbox/
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca) and Michael Mandel
% (mim@mr-pc.org), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Fix for ending zeros that mess up the plot
if trace(2,end)==0
trace=trace(:,1:end-1);
end
if trace(2, end-1)==0
trace(2,end-1)=trace(2,end-2);
end

% hop size between frames
stftHop = 0.025;

% read midi file
nmat=readmidi(mid);

% plot spectogram of audio file
imagesc(20*log10(spec));
title(['Spectrogram with Aligned MIDI Notes Overlaid']);
xlabel(['Time (.05s)']);
ylabel(['Midinote']);
axis xy;
caxis(max(caxis)+[-50 0])
colormap(1-gray)

% zoom in fundamental frequencies
notes = nmat(:,4)';
notes = (2.^((notes-105)/12))*440;
notes(end+1) = notes(end);
nlim = length(notes);

% plot alignment
plotFineAlign(trace(1,:), trace(2,:), notes(1:nlim), stftHop);
if size(trace,1) >= 3
notenums = trace(3,2:end);
else
nlim = length(notes);
notenums = [reshape(repmat(1:nlim,4,1),1,[]) nlim];
end


function plotFineAlign(stateType, occupancy, notes, stftHop)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% plotFineAlign(stateType, occupancy, notes, stftHop, highlight)
%
% Description:
% Plot the HMM alignment based on the output of YIN. StateType is the
% list of states in the HMM, and occupancy is the number of YIN frames
% for which that state is occupied. Notes is a list of midi note numbers
% that are played, should be one note for each [3] in stateType. If the
% highlight vector is supplied, it should contain indices of the states
% to highlight by plotting an extra line at the bottom of the window.
%
% Inputs:
% stateType - vector with a list of states
% occupancy - vector indicating the time (in seconds) at which the states
% in stateType end
% notes - vector of notes from MIDI file
% stftHop - the hop size between frames in the spectrogram
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca) and Michael Mandel
% (mim@mr-pc.org), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Plot the 4 states: silence in red, beginning transient in green,
% steady state in blue, ending transient in green.

styles = {{'r+-', 'LineWidth', 2},
{'g+-', 'LineWidth', 2},
{'b+-', 'LineWidth', 2}};

cs = occupancy /stftHop;
segments = [cs(1:end-1); cs(2:end)]';

hold on

stateNote = max(1, cumsum(stateType == 3)+1);
for i=1:size(segments,1)
plot(segments(i,:)', repmat(notes(stateNote(i)),2,1), styles{stateType(i+1)}{:})
end

hold off
Binary file added example.mid
View file
Binary file not shown.
Binary file added example.wav
View file
Binary file not shown.
72 changes: 72 additions & 0 deletions exampleScript.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% exampleScript.m
%
% Description:
% Example of how to use the HMM alignment algorithm
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% You will need to have the following toolkits installed and in your path
% de Cheveigné, A. 2002. YIN MATLAB implementation Available from:
% http://audition.ens.fr/adc/sw/yin.zip
% Ellis, D. P. W. 2003. Dynamic Time Warp (DTW) in Matlab. Available
% from: http://www.ee.columbia.edu/~dpwe/resources/matlab/dtw/
% Ellis, D. P. W. 2008. Aligning MIDI scores to music audio. Available
% from: http://www.ee.columbia.edu/~dpwe/resources/matlab/alignmidiwav/
% Murphy, K. 1998. Hidden Markov Model (HMM) Toolbox for Matlab.
% Available from http://www.cs.ubc.ca/~murphyk/Software/HMM/hmm.html
% Toiviainen, P. and T. Eerola. 2006. MIDI Toolbox. Available from:
% https://www.jyu.fi/hum/laitokset/musiikki/en/research/coe/materials
% /miditoolbox/
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% audio file to be aligned
audiofile=('example.wav');

% MIDI file to be aligned
midifile=('example.mid');

% number of notes to align
numNotes=6;

% vector of order of states (according to lyrics) in stateOrd and
% corresponding note numbers in noteNum
% 1 indicates a rest at the beginning of ending of the note
% 2 indicates a transient at the beginning or ending of the note
% 3 indicates a steady state section
% the following encoding is for six syllables "A-ve Ma-ri-(i)-a"
% syllable A-ve Ma-ri-(i)-a
% state type 13 23 23 23 3 31
% note number 11 22 33 44 5 66
stateOrd = [1 3 2 3 2 3 2 3 3 3 1];
noteNum = [1 1 2 2 3 3 4 4 5 6 6];

% load singing means and covariances for the HMM alignment
load SingingMeansCovars.mat
means=sqrtmeans;
covars=sqrtcovars;

% specify that the means and covariances in the HMM won't be learned
learnparams=0;

% run the alignment
[allstate selectstate,spec,yinres]=runAlignment(audiofile, midifile, numNotes, stateOrd, noteNum, means, covars, learnparams);

% visualise the alignment
alignmentVisualiser(selectstate,midifile,spec,1);

% get onset and offset times
times=getOnsOffs(selectstate);

% write the onset and offset times to an audacity-readable file
dlmwrite('example.txt',[times.ons' times.offs'], 'delimiter', '\t');

% map timing information to the quantized MIDI file
nmatNew=getTimingData(midifile, times);

% calculate intervals size, perceived pitch, vibrato rate, vibrato depth, and loudness
[vibratoDepth, vibratoRate, noteDynamics, intervalSize, pp,nmatNew]=getPitchVibratoDynamicsData(times,yinres,nmatNew);
77 changes: 77 additions & 0 deletions exampleScript.m~
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% exampleScript.m
%
% Description:
% Example of how to use the HMM alignment algorithm
%
% Automatic Music Performance Analysis and Analysis Toolkit (AMPACT)
% http://www.ampact.org
% (c) copyright 2011 Johanna Devaney (j@devaney.ca), all rights reserved.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% you will need to have the following toolkits installed and in your path
% de Cheveign�, A. 2002. YIN MATLAB implementation Available from:
% http://audition.ens.fr/adc/sw/yin.zip
% Ellis, D. P. W. 2003. Dynamic Time Warp (DTW) in Matlab. Available
% from: http://www.ee.columbia.edu/~dpwe/resources/matlab/dtw/
% Ellis, D. P. W. 2008. Aligning MIDI scores to music audio. Available
% from: http://www.ee.columbia.edu/~dpwe/resources/matlab/alignmidiwav/
% Murphy, K. 1998. Hidden Markov Model (HMM) Toolbox for Matlab.
% Available from http://www.cs.ubc.ca/~murphyk/Software/HMM/hmm.html
% Toiviainen, P. and T. Eerola. 2006. MIDI Toolbox. Available from:
% https://www.jyu.fi/hum/laitokset/musiikki/en/research/coe/materials
% /miditoolbox/

% audio file to be aligned
audiofile=('example.wav');

% MIDI file to be aligned
midifile=('example.mid');

% number of notes to align
numNotes=6;

% vector of order of states (according to lyrics) in stateOrd and
% corresponding note numbers in noteNum
% 1 indicates a rest at the beginning of ending of the note
% 2 indicates a transient at the beginning or ending of the note
% 3 indicates a steady state section
% the following encoding is for six syllables "A-ve Ma-ri-(i)-a"
% syllable A-ve Ma-ri-(i)-a
% state type 13 23 23 23 3 31
% note number 11 22 33 44 5 66
stateOrd = [1 3 2 3 2 3 2 3 3 3 1];
noteNum = [1 1 2 2 3 3 4 4 5 6 6];

% load singing means and covariances for the HMM alignment
load SingingMeansCovars.mat
means=sqrtmeans;
covars=sqrtcovars;

% specify that the means and covariances in the HMM won't be learned
learnparams=0;

% run the alignment
[allstate selectstate,spec,yinres]=runAlignment(audiofile, midifile, numNotes, stateOrd, noteNum, means, covars, learnparams);

% visualise the alignment
alignmentVisualiser(selectstate,midifile,spec);

% get onset and offset times
times=getOnsOffs(selectstate);

% map timing information to the quantized MIDI file
nmatNew=getTimingData(midifile, times)
% visualise the between the quantized version and actual performance using the pianoroll
% function from the MIDI Toolbox
figure(1)
subplot(211)
pianoroll(nmat,'b','vel')
figure(3)
pianoroll(nmatOld,'b','vel')

% calculate intervals size, perceived pitch, vibrato rate, vibrato depth, and loudness
[vibratoDepth, vibratoRate, noteDynamics, intervalSize, pp]=getPitchVibratoDynamicsData(times,yinres)



0 comments on commit ac6cbab

Please sign in to comment.