Skip to content
This repository has been archived by the owner on Dec 4, 2018. It is now read-only.

Commit

Permalink
sliding window
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Sloan committed Mar 1, 2011
1 parent d2ec6e0 commit cb3439f
Show file tree
Hide file tree
Showing 3 changed files with 586 additions and 151 deletions.
236 changes: 85 additions & 151 deletions chordal/main.cpp
View file
Expand Up @@ -8,24 +8,17 @@
using namespace ntk;
using namespace cv;

float logeto2 = 1 / log (2);

static const double pi = 3.14159265358979323846;

inline static double square(int a) { return a * a; }

void extractSpectrum(Mat img, Mat spect, // depth input, spectrum output
int x, int fy, int ty, // line in camera space, maps to
int ff, int tf, // interval in frequency.
float fd, float td, // interval in distance, maps to
float fa, float ta, // interval in amplitude.
bool logScale) {

float deltaF = logScale ? log ((float) tf / ff) * logeto2
float deltaF = logScale ? log ((float) tf / ff)
: tf - ff;
float fratio = deltaF / (ty - fy),
aratio = (ta - fa) / (td - fd);
float logfrom = log(ff) * logeto2;
bool prev = false;
for (int y = fy; y <= ty; y++) {
// calculate amplitude of value.
Expand All @@ -38,11 +31,11 @@ void extractSpectrum(Mat img, Mat spect, // depth input, spectrum output
//val = max(fd, val);
float amp = (td - val) * aratio + fa;

float freq = (y - fy) * fratio;
freq *= logScale ? pow(2, freq + logfrom)
: freq + ff;

// calculate frequency / increment appropriate index in spectrum.
float freq = (y - fy) * fratio;
freq = logScale ? exp(freq + log(ff))
: freq + ff;
//printf("%f\n", freq);
spect.at<float>(0, (int)freq) += amp;
}
}
Expand All @@ -60,45 +53,16 @@ void drawSpectrum(Mat img, Mat spect, int x, int fy, int ty) {
}
}

Mat getIntervals(Mat slice, float ratio) {
vector<float> results;
int height = slice.size().height;
int prev_y = -1;
float start_val;
for (int y = 0; y < height; y++) {
float value = slice.at<float>(y, 0);
if (prev_y != -1) {
if (value < 0.4 || fabs(value - start_val) > ratio * start_val) {
// printf("%f %i %i\n", start_val, prev_y, y - 1);
results.push_back(start_val);
results.push_back(prev_y);
results.push_back(y - 1);
prev_y = -1;
}
} else if (value > 0.4) {
prev_y = y;
start_val = value;
}
}
Mat result(results, true);
result.reshape(0, results.size() / 3);
return result;
}

int scaleValueLog(float y, float fy, float ty, float ff, float octaves) {
float deltaF = log (octaves * 2) * logeto2;
float freq = (y - fy) * (deltaF / (ty - fy));
return pow(2, freq + log(ff) * logeto2);
}

int scaleValueLin(float y, float fy, float ty, float ff, float tf) {
return (y - fy) * ((tf - ff) / (ty - fy)) + ff;
}
timeval tv;
long startsec;

int main() {
int res = libao_start();
if (res) return res;

gettimeofday(&tv, NULL);
startsec = tv.tv_sec;

KinectGrabber grabber;
grabber.initialize();

Expand All @@ -111,27 +75,36 @@ int main() {
processor.setFilterFlag(RGBDProcessor::ComputeKinectDepthBaseline, true);

// OpenCV windows.
// namedWindow("color");
// namedWindow("depth");
namedWindow("dac");

int scale = 10, fromIx = 10, toIx = 1000;
int scale = 10;
int sound_width = sound.size().width;
int ssize = sound_width * 2;

Mat spectrum(1, ssize, CV_32F);
Mat prev_result(1, ssize, CV_32F);
Mat prev_slice(480, 1, CV_32F);
//for (int i = 0; i < ssize; i++) spectrum.at<float>(0, i) = 0;
for (int i = 0; i < 480; i++) prev_slice.at<float>(i, 0) = 0;
for (int i = 0; i < ssize; i++) spectrum.at<float>(0, i) = 0;

Mat old_frame, flow;

// Current image. An RGBDImage stores rgb and depth data.
RGBDImage current_frame;
Mat first_frame;
bool got_first = false;
while (true)
{
cvCreateTrackbar("scale", "dac", &scale, 100, NULL);
cvCreateTrackbar("from", "dac", &fromIx, ssize, NULL);
// cvCreateTrackbar("to", "dac", &toIx, ssize, NULL);

/*
int wid = sound.size().width;
for (int i = 0; i < sound.size().width; i++) {
float frac = (float)i / wid;
float coef = frac < 0.05 ? frac : (frac > 0.95 ? (1 - frac) : 0.05);
sound.at<short>(0, i) = (short)(sin((float)i / 10.0) * 0x2010 * coef * 20 + 0x3010);
}
*/

grabber.waitForNextFrame();
grabber.copyImageTo(current_frame);
Expand All @@ -143,112 +116,59 @@ int main() {
cv::format("%d fps", fps),
Point(10,20), 0, 0.5, Scalar(255,0,0,255));

// Compute color encoded depth.
cv::Mat3b dac;
compute_color_encoded_depth(current_frame.depth(), dac);

Mat depth = current_frame.depth();

Mat slice = depth.col(120), slmask, dy;
compare(slice, 0.4, slmask, CMP_GT);

Mat ivls = getIntervals(slice, 0.5);
int ivlCount = ivls.size().height;
vector<float> pos;
for (int i = 0; i < ivlCount; i++) {
float value = ivls.at<float>(i, 0),
from = ivls.at<float>(i, 1),
to = ivls.at<float>(i, 2);
if (value < 1 && to - from > 2) {
float mean = (from + to) / 2;
// spectrum.at<float>(0, (int)mean) = 100;
pos.push_back(mean);
//printf("%i ", (int)mean);
}
}

float height = depth.size().height, octaves = 2;
float spacing = height / (octaves * 12.0);
for (int i = 0; i < octaves * 12; i++) {
line(dac, Point(150, i * spacing), Point(155, i * spacing), Scalar(0,0,0));
}
for (int i = 0; i < pos.size(); i++) {
float val = floor(pos[i] / spacing);
float ffreq = scaleValueLog(val, 0, 24, fromIx, octaves);
spectrum.at<float>(0, ffreq) = 100;
}
if (!got_first) first_frame = depth;

/*
vector<Mat> rgbplanes;
split(current_frame.rgb(), rgbplanes);
Mat frame = rgbplanes[1];
if (!old_frame.empty()) {
int flags = 0; //flow.empty() ? 0 : OPTFLOW_USE_INITIAL_FLOW;
calcOpticalFlowFarneback(old_frame, frame, flow, 0.5, 3, 3, 3, 5, 1.1, flags);
vector<Mat> planes;
split(flow, planes);
planes[0] += 20;
planes[1] += 20;
planes[0] /= 40;
planes[1] /= 40;
imshow("flowx", planes[0]);
imshow("flowy", planes[1]);
}
frame.copyTo(old_frame);
*/


/*
if (pos.size() >= 1) {
float height = depth.size().height, octaves = 2;
float val = floor(pos[0] * (octaves * 12.0) / height);
float ffreq = scaleValueLog(val, 0, 24, fromIx, octaves);
printf("%f \n", ffreq);
for (int j = 1; j < 2; j++) {
int fund = ffreq * j;
if (fund > ssize) break;
//if (pos.size() == 1)
spectrum.at<float>(0, fund) = 100 / j;
if (pos.size() >= 2) {
int fifth = ffreq * log (4.0 / 3.0) * logeto2 * j;
if (fifth < ssize)
spectrum.at<float>(0, fifth) = 100 / j;
}
if (pos.size() > 2) {
if (pos[1] - pos[0] > pos[2] - pos[1]) {
int third = ffreq * 5.0 / 4.0 * j;
if (third < ssize)
spectrum.at<float>(0, third) = 100 / j;
// printf("major\n");
} else {
int third = ffreq * 6.0 / 5.0 * j;
if (third < ssize)
spectrum.at<float>(0, third) = 100 / j;
// printf("minor\n");
}
} else {
}
}
}
*/

//spectrum.at<float>(0, (int) mean) = (0.7 - value) * 300;

// Sobel(depth, dy, CV_32F, 0, 1);
// imshow("dy", dy);
Mat slice = depth.col(150);
Mat x, acc;
compare(slice, 0.4, x, CMP_LT);
compare(prev_slice, 0.4, acc, CMP_LT);
bitwise_or(x, acc, acc);
compare(prev_slice, 2, x, CMP_GT);
bitwise_or(x, acc, acc);
compare(slice, 2, x, CMP_GT);
bitwise_or(x, acc, acc);
Mat diff = slice - prev_slice;
// diff.setTo(0, acc);
// prev_slice = slice;
slice.copyTo(prev_slice);
extractSpectrum(diff, spectrum,
0, 1, 479, // line
10, ssize, // frequency
-0.2, 0.2, // distance
-50, 50, // amplitude
true);
compare(spectrum, 0, x, CMP_LT);
spectrum.setTo(0, x);
*/

gettimeofday(&tv, NULL);
int elapsed = tv.tv_sec - startsec;
float time = (float)tv.tv_usec * 0.000001 + (float)elapsed;
printf("%f\n", time);
int xpos = ((int)(time * 10.0)) % 640;

extractSpectrum(first_frame, spectrum,
xpos, 1, 479, // line
10, ssize, // frequency
0.4, 2, // distance
0, 50, // amplitude
true);

// Compute color encoded depth.
cv::Mat3b dac;
compute_color_encoded_depth(first_frame, dac);

line(dac, Point(150, 0), Point(150, 480), Scalar(0,0,0));
drawSpectrum(dac, spectrum, xpos, 1, 479);

// for (int i = -4; i < 5; i++) {
// int ypos = minLoc.y + i * 30;
// line(dac, Point(150, ypos), Point(160, ypos), Scalar(64,64,64));
// }

drawSpectrum(dac, spectrum, 150, 1, 479);
//for (int i = ssize / 2; i < ssize; i++) spectrum.at<float>(0, i) = 0;

Mat result;
dft(spectrum, result, DFT_INVERSE);
for (int i = ssize / 2; i < ssize; i++) spectrum.at<float>(0, i) = 0;

float prev = 0;
for (int i = 0; i < sound_width; i++) {
Expand All @@ -265,12 +185,26 @@ int main() {
line(dac, Point(prev_xpos, prev / 2 + 101),
Point(xpos, val / 2 + 101), Scalar(0,0,0));
sound.at<short>(0,i) = (short)(val * 10 + 16000);
// printf("%ih ", sound.at<short>(0,i));
prev = val;
}
spectrum *= 0;
spectrum *= 0.9;

/*
for (int i = 0; i < sound_width; i++) {
psound.at<short>(0,i) = sin()
}
*/

result.copyTo(prev_result);

// Display the color image
// imshow("color", current_frame.rgb());

// Show the depth image as normalized gray scale
// imshow_normalized("depth", current_frame.depth());


imshow("dac", dac);

// Enable switching to InfraRead mode.
Expand Down

0 comments on commit cb3439f

Please sign in to comment.