-
Notifications
You must be signed in to change notification settings - Fork 2
/
fishMusicDemo.m
58 lines (48 loc) · 1.85 KB
/
fishMusicDemo.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
% Demonstrates generation of music from video of fish
movie = 'lq.mp4';
beatDuration = .2;
% estimate optical flow for first 30 seconds of movie
flow = getOpticalFlow(movie, beatDuration, 30);
videoReader = VideoReader(movie);
% magnitude of optical flow
magnitude = squeeze(sqrt(flow(:,:,1,:).^2 + flow(:,:,2,:).^2));
% normalize optical flow so that max magnitude is 1
flow = flow/max(magnitude(:));
% recalculate magnitude
magnitude = squeeze(sqrt(flow(:,:,1,:).^2 + flow(:,:,2,:).^2));
% data2Music argument vectors
beats = [];
x = [];
y = [];
velocity = [];
dominant = [];
% for every frame of optical flow calculated...
for i = 1:size(flow, 4)
% produce the smoothed magnitude of the flow (median, 5x5 neighborhood)
smoothed = medfilt2(magnitude(:,:,i),[5 5]);
% r, c: row, col indices of local peaks of the magnitude
[r,c] = find(smoothed > imdilate(smoothed, [1 1 1; 1 0 1; 1 1 1]));
% for every index...
for j=1:length(r)
% linearly scale the peak magnitude to a MIDI velocity
vel = min([round(smoothed(r(j), c(j)) * 1000), 127]);
% if this is sufficiently high, add this data point
if (vel > 5)
beats = [beats i];
% add the normalized column to the x vector
x = [x ((c(j) - 1) / (videoReader.Width - 1) - .5) * 2];
% add the normalized row to the y vector
y = [y -((r(j) - 1) / (videoReader.Height - 1) - .5) * 2];
% dominant reflects the direction of the flow
dominant = [dominant flow(:,:,1,i)>0];
% add vel to the velocity vector
velocity = [velocity vel];
end
end
end
% produce a music matrix
music = data2Music(beats, x*2, y*2, velocity, dominant);
% filter out repeated notes due to continuous motion
music = filterRepeats(music, 5);
% play it
play(music, beatDuration, pickMidiReceiver)