Skip to content

Commit

Permalink
let user control alignment direction
Browse files Browse the repository at this point in the history 
- alignment direction CLI argument
- YUYV/RGBA for color/depth alignment direction
   - librealsense not supporting YUYV alignment to depth
   - but YUYV is native Realsense format (optimal)
   - so we use it when possible (aligning to color)
- determine result resolutions on alignment direction

Related to #15
  • Loading branch information
@bmegli
bmegli committed Jun 5, 2020
1 parent a37879e commit b116ab4
Showing 1 changed file with 81 additions and 57 deletions.
138 changes: 81 additions & 57 deletions rnhve_depth_color.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Realsense Network Hardware Video Encoder
*
* Realsense hardware encoded UDP HEVC multi-streaming
* Realsense hardware encoded UDP HEVC aligned multi-streaming
* - depth (Main10) + color (Main)
*
* Copyright 2020 (C) Bartosz Meglicki <meglickib@gmail.com>
Expand All @@ -27,6 +27,9 @@ using namespace std;

int hint_user_on_failure(char *argv[]);

//encoding index, alignment direction
enum Stream {Depth = 0, Color = 1};

//user supplied input
struct input_args
{
Expand All @@ -37,6 +40,7 @@ struct input_args
int framerate;
int seconds;
float depth_units;
Stream align_to;
};

bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer);
Expand All @@ -46,9 +50,6 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi

const uint16_t P010LE_MAX = 0xFFC0; //in binary 10 ones followed by 6 zeroes

const int DEPTH = 0; //depth hardware encoder index
const int COLOR = 1; //color hardware encoder index

int main(int argc, char* argv[])
{
//prepare NHVE Network Hardware Video Encoder
Expand Down Expand Up @@ -88,12 +89,12 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer

uint16_t *depth_uv = NULL; //data of dummy color plane for P010LE

rs2::align align_to_color(RS2_STREAM_COLOR);
rs2::align aligner( (input.align_to == Color) ? RS2_STREAM_COLOR : RS2_STREAM_DEPTH);

for(f = 0; f < frames; ++f)
{
rs2::frameset frameset = realsense.wait_for_frames();
frameset = align_to_color.process(frameset);
frameset = aligner.process(frameset);

rs2::depth_frame depth = frameset.get_depth_frame();
rs2::video_frame color = frameset.get_color_frame();
Expand Down Expand Up @@ -138,12 +139,13 @@ bool main_loop(const input_args& input, rs2::pipeline& realsense, nhve *streamer
void init_realsense(rs2::pipeline& pipe, const input_args& input)
{
rs2::config cfg;
//use YUYV/RGBA when aligning to color/depth (aligning YUYV not possible in librealsense)
rs2_format color_format = (input.align_to == Color) ? RS2_FORMAT_YUYV : RS2_FORMAT_RGBA8;

cfg.enable_stream(RS2_STREAM_DEPTH, input.depth_width, input.depth_height, RS2_FORMAT_Z16, input.framerate);
cfg.enable_stream(RS2_STREAM_COLOR, input.color_width, input.color_height, RS2_FORMAT_YUYV, input.framerate);
cfg.enable_stream(RS2_STREAM_COLOR, input.color_width, input.color_height, color_format, input.framerate);

rs2::pipeline_profile profile = pipe.start(cfg);

rs2::depth_sensor depth_sensor = profile.get_device().first<rs2::depth_sensor>();

try
Expand Down Expand Up @@ -179,8 +181,10 @@ void init_realsense(rs2::pipeline& pipe, const input_args& input)

cout << "Clamping range at " << input.depth_units * P010LE_MAX << " m" << endl;

print_intrinsics(profile, RS2_STREAM_DEPTH);
print_intrinsics(profile, RS2_STREAM_COLOR);
if(input.align_to == Color)
print_intrinsics(profile, RS2_STREAM_COLOR);
else
print_intrinsics(profile, RS2_STREAM_DEPTH);
}

void print_intrinsics(const rs2::pipeline_profile& profile, rs2_stream stream)
Expand All @@ -201,28 +205,44 @@ void print_intrinsics(const rs2::pipeline_profile& profile, rs2_stream stream)

int process_user_input(int argc, char* argv[], input_args* input, nhve_net_config *net_config, nhve_hw_config *hw_config)
{
if(argc < 9)
if(argc < 10)
{
cerr << "Usage: " << argv[0] << " <host> <port> <width_depth> <height_depth> <width_color> <height_color>" << endl
<< " <framerate> <seconds> [device] [bitrate_depth] [bitrate_color] [depth units]" << endl;
cerr << "Usage: " << argv[0] << endl
<< " <host> <port>" << endl //1, 2
<< " <color/depth> # alignment direction" << endl //3
<< " <width_depth> <height_depth> <width_color> <height_color>" << endl //4, 5, 6, 7
<< " <framerate> <seconds>" << endl //8, 9
<< " [device] [bitrate_depth] [bitrate_color] [depth units]" << endl; //10, 11, 12, 13

cerr << endl << "examples: " << endl;
cerr << argv[0] << " 127.0.0.1 9766 640 360 640 360 30 5" << endl;
cerr << argv[0] << " 127.0.0.1 9766 640 360 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 192.168.0.125 9766 640 360 640 360 30 50 /dev/dri/renderD128 4000000 1000000" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125f" << endl;
cerr << argv[0] << " 192.168.0.100 9768 848 480 1280 720 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125f" << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 640 360 30 5" << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 192.168.0.125 9766 color 640 360 640 360 30 50 /dev/dri/renderD128 4000000 1000000" << endl;
cerr << argv[0] << " 192.168.0.100 9768 color 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0001" << endl;
cerr << argv[0] << " 192.168.0.100 9768 color 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00005" << endl;
cerr << argv[0] << " 192.168.0.100 9768 color 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.000025" << endl;
cerr << argv[0] << " 192.168.0.100 9768 color 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.0000125" << endl;
cerr << argv[0] << " 192.168.0.100 9768 color 848 480 848 480 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125f" << endl;
cerr << argv[0] << " 192.168.0.100 9768 depth 848 480 1280 720 30 500 /dev/dri/renderD128 8000000 1000000 0.00003125f" << endl;

return -1;
}

net_config->ip = argv[1];
net_config->port = atoi(argv[2]);

char c = argv[3][0]; //color, depth
if(c == 'c') input->align_to = Color;
else if(c == 'd') input->align_to = Depth;
else
{
cerr << "unnkown alignment target '" << argv[3] <<"', valid targets: 'color', 'depth'" << endl;
return -1;
}

cout << "Aligning to " << ((input->align_to == Color) ? "color" : "depth") << endl;

//for depth encoding we use 10 bit P010LE pixel format
//that can be directly matched with Realsense output as P016LE Y plane
//with precision/range trade-off controlled by Realsense Depth Units
Expand All @@ -231,71 +251,75 @@ int process_user_input(int argc, char* argv[], input_args* input, nhve_net_confi

//native format of Realsense RGB sensor is YUYV (YUY2, YUYV422)
//see https://github.com/IntelRealSense/librealsense/issues/3042
//we will match:
//- Realsense RGB sensor YUYV with VAAPI YUYV422 (same format)
//however librealsense is unable to align color with YUYV to depth
//see https://github.com/IntelRealSense/librealsense/blob/master/src/proc/align.cpp#L123

//for both cases (depth and color) we use native hardware formats
//and there is no need for data processing on the host CPU
//we will match:
//- Realsense RGB sensor YUYV with VAAPI YUYV422 (same format) when aligning to color
//- Realsense RGB sensor RGBA8 with VAAPI RGB0 (alpha ignored) when aligning to depth

input->depth_width = atoi(argv[3]);
input->depth_height = atoi(argv[4]);
input->color_width = atoi(argv[5]);
input->color_height = atoi(argv[6]);
input->depth_width = atoi(argv[4]);
input->depth_height = atoi(argv[5]);
input->color_width = atoi(argv[6]);
input->color_height = atoi(argv[7]);

//DEPTH hardware encoding configuration
hw_config[DEPTH].profile = FF_PROFILE_HEVC_MAIN_10;
hw_config[DEPTH].pixel_format = "p010le";
hw_config[DEPTH].encoder = "hevc_vaapi";
hw_config[Depth].profile = FF_PROFILE_HEVC_MAIN_10;
hw_config[Depth].pixel_format = "p010le";
hw_config[Depth].encoder = "hevc_vaapi";

//dimmesions of depth match color after alignmnet
hw_config[DEPTH].width = input->color_width;
hw_config[DEPTH].height = input->color_height;
//dimmensions will match alignment target
hw_config[Depth].width = (input->align_to == Color) ? input->color_width : input->depth_width;
hw_config[Depth].height = (input->align_to == Color) ? input->color_height : input->depth_height;

hw_config[DEPTH].framerate = input->framerate = atoi(argv[7]);
hw_config[Depth].framerate = input->framerate = atoi(argv[8]);

input->seconds = atoi(argv[8]);
input->seconds = atoi(argv[9]);

hw_config[DEPTH].device = argv[9]; //NULL as last argv argument, or device path
hw_config[Depth].device = argv[10]; //NULL as last argv argument, or device path

if(argc > 10)
hw_config[DEPTH].bit_rate = atoi(argv[10]);
if(argc > 11)
hw_config[Depth].bit_rate = atoi(argv[11]);

//COLOR hardware encoding configuration
hw_config[COLOR].profile = FF_PROFILE_HEVC_MAIN;
hw_config[COLOR].pixel_format = "yuyv422";
hw_config[COLOR].encoder = "hevc_vaapi";
hw_config[Color].profile = FF_PROFILE_HEVC_MAIN;
//use YUYV/RGBA when aligning to color/depth (aligning YUYV not possible in librealsense)
hw_config[Color].pixel_format = (input->align_to == Color) ? "yuyv422" : "rgb0";
hw_config[Color].encoder = "hevc_vaapi";

hw_config[COLOR].width = input->color_width;
hw_config[COLOR].height = input->color_height;
hw_config[COLOR].framerate = input->framerate = atoi(argv[7]);
//dimmensions will match alignment target
hw_config[Color].width = (input->align_to == Color) ? input->color_width : input->depth_width;
hw_config[Color].height = (input->align_to == Color) ? input->color_height : input->depth_height;

hw_config[COLOR].device = argv[9]; //NULL as last argv argument, or device path
hw_config[Color].framerate = input->framerate = atoi(argv[8]);

if(argc > 11)
hw_config[COLOR].bit_rate = atoi(argv[11]);
hw_config[Color].device = argv[10]; //NULL as last argv argument, or device path

if(argc > 12)
hw_config[Color].bit_rate = atoi(argv[12]);

//set highest quality and slowest encoding
//this adds around 3 ms and 10% GPU usage on my 2017 KabyLake
//with 848x480 HEVC Main10 encoding
hw_config[DEPTH].compression_level = 1;
hw_config[COLOR].compression_level = 0;
hw_config[Depth].compression_level = 1;
hw_config[Color].compression_level = 0;

//optionally set qp instead of bit_rate for CQP mode
//hw_config[].qp = ...

//optionally set gop_size (determines keyframes period)
//hw_config[].gop_size = ...;

if(argc > 12)
input->depth_units = strtof(argv[12], NULL);
if(argc > 13)
input->depth_units = strtof(argv[13], NULL);

return 0;
}

int hint_user_on_failure(char *argv[])
{
cerr << "unable to initalize, try to specify device e.g:" << endl << endl;
cerr << argv[0] << " 127.0.0.1 9766 640 360 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 640 360 640 360 30 5 /dev/dri/renderD129" << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 640 360 30 5 /dev/dri/renderD128" << endl;
cerr << argv[0] << " 127.0.0.1 9766 color 640 360 640 360 30 5 /dev/dri/renderD129" << endl;
return -1;
}

0 comments on commit b116ab4

Please sign in to comment.