Skip to content

Commit

Permalink
Merge pull request os-autoinst#2417 from marmarek/video-formats
Browse files Browse the repository at this point in the history 
Add more video formats/methods
  • Loading branch information
@mergify
mergify[bot] committed Jan 11, 2024
2 parents a45f826 + 0195131 commit 5eb8a49
Show file tree
Hide file tree
Showing 18 changed files with 433 additions and 40 deletions.
3 changes: 2 additions & 1 deletion backend/generalhw.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ sub relogin_vnc ($self) {
port => $bmwqemu::vars{GENERAL_HW_VNC_PORT} // 5900,
password => $bmwqemu::vars{GENERAL_HW_VNC_PASSWORD},
depth => $bmwqemu::vars{GENERAL_HW_VNC_DEPTH} // 16,
connect_timeout => 50
connect_timeout => 50,
jpeg => $bmwqemu::vars{GENERAL_HW_VNC_JPEG} // 0,
});
$vnc->backend($self);
$self->select_console({testapi_console => 'sut'});
Expand Down
96 changes: 84 additions & 12 deletions consoles/VNC.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ has [qw(description hostname port username password socket name width height dep
no_endian_conversion _pixinfo _colourmap _framebuffer _rfb_version screen_on
_bpp _true_colour _do_endian_conversion absolute ikvm keymap _last_update_received
_last_update_requested check_vnc_stalls _vnc_stalled vncinfo old_ikvm dell
vmware_vnc_over_ws_url original_hostname)];
vmware_vnc_over_ws_url original_hostname jpeg)];

our $VERSION = '0.40';

Expand Down Expand Up @@ -83,24 +83,39 @@ my %supported_depths = (
},
);

# Entries in order of preference
my @encodings = (

# These ones are defined in rfbproto.pdf
{
num => 16,
name => 'ZRLE',
supported => 1,
},
{
num => 0,
name => 'Raw',
supported => 1,
},
{
num => 16,
name => 'ZRLE',
num => 7,
name => 'Tight',
supported => 1,
},
{
num => -24,
name => 'JPEG quality',
supported => 1,
},
{
num => -223,
name => 'DesktopSize',
supported => 1,
},
{
num => -224,
name => 'VNC_ENCODING_LAST_RECT',
supported => 1,
},
{
num => -257,
name => 'VNC_ENCODING_POINTER_TYPE_CHANGE',
Expand All @@ -111,11 +126,6 @@ my @encodings = (
name => 'VNC_ENCODING_LED_STATE',
supported => 1,
},
{
num => -224,
name => 'VNC_ENCODING_LAST_RECT',
supported => 1,
},
);

sub login ($self, $connect_timeout = undef, $timeout = undef) {
Expand Down Expand Up @@ -428,14 +438,16 @@ sub _server_initialization ($self) {

my @encs = grep { $_->{supported} } @encodings;

# Prefer the higher-numbered encodings
@encs = reverse sort { $a->{num} <=> $b->{num} } @encs;

if ($self->dell) {
# idrac's ZRLE implementation even kills tigervnc, they duplicate
# frames under certain conditions. Raw works ok
@encs = grep { $_->{name} ne 'ZRLE' } @encs;
}
if (!$self->jpeg) {
# according to the spec, RAW encoding is least preferred, so don't let
# loosy Tight JPEG be used over RAW unless explicitly requested
@encs = grep { $_->{name} ne 'Tight' and $_->{name} ne 'JPEG quality' } @encs;
}
$socket->print(
pack(
'CCn',
Expand Down Expand Up @@ -840,6 +852,9 @@ sub _receive_update ($self) {
$socket->read(my $data, $w * $h * $self->_bpp / 8) || die 'unexpected end of data';
$image->map_raw_data($data, $x, $y, $w, $h, $self->vncinfo);
}
elsif ($encoding_type == 7) { # Tight
$self->_receive_tight_encoding($x, $y, $w, $h);
}
elsif ($encoding_type == 16) { # ZRLE
$self->_receive_zrle_encoding($x, $y, $w, $h);
}
Expand Down Expand Up @@ -921,6 +936,63 @@ sub _receive_zrle_encoding ($self, $x, $y, $w, $h) {
return $res;
}

# wrapper to make testing easier
sub _read_socket ($socket, $data, $data_len, $offset) { return read($socket, $data, $data_len, $offset); } # uncoverable statement

sub _receive_tight_encoding ($self, $x, $y, $w, $h) {
my $socket = $self->socket;
my $image = $self->_framebuffer;

$socket->read(my $data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
my ($compression_control) = unpack('C', $data);
# FillCompression
if (($compression_control & 0xF0) == 0x80) {
my $data;
# special case for TPIXEL, otherwise identical to PIXEL
if ($self->_true_colour and $self->_bpp == 32 and $self->depth == 24) {
$socket->read($data, 3)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
$data = pack('CCCx', unpack('CCC', $data));
} else {
$socket->read($data, $self->_bpp / 8)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
}
$image->fill_pixel($data, $self->vncinfo, $x, $y, $w, $h);
return;
}
# Only Fill (above) and JPEG for now; "Basic" unsupported
die "Unsupported compression $compression_control" if ($compression_control & 0xF0) != 0x90;

$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len) = unpack('C', $data);
if ($data_len & 0x80) {
$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len2) = unpack('C', $data);
$data_len = ($data_len & 0x7f) | ($data_len2 & 0x7f) << 7;
if ($data_len2 & 0x80) {
$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len2) = unpack('C', $data);
$data_len |= $data_len2 << 14;
}
}

my $read_len = 0;
while ($read_len < $data_len) {
my $len = _read_socket($socket, $data, $data_len - $read_len, $read_len);
OpenQA::Exception::VNCProtocolError->throw(error => "short read for jpeg data $read_len - $data_len") unless $len;
$read_len += $len;
}
my $rect = tinycv::from_ppm($data);
OpenQA::Exception::VNCProtocolError->throw(error => "Invalid width/height of the rectangle (${w}x${h} != " . $rect->xres . "x" . $rect->yres . ")")
unless $w == $rect->xres and $h == $rect->yres;
$image->blend($rect, $x, $y);
$self->_framebuffer($image);
}

sub _receive_ikvm_encoding ($self, $encoding_type, $x, $y, $w, $h) {
my $socket = $self->socket;
my $image = $self->_framebuffer;
Expand Down
168 changes: 144 additions & 24 deletions consoles/video_stream.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ package consoles::video_stream;
use Mojo::Base 'consoles::video_base', -signatures;
use Mojo::UserAgent;
use Mojo::URL;
use Mojo::Util 'scope_guard';

use List::Util 'max';
use Time::HiRes qw(usleep);
use Time::HiRes qw(usleep clock_gettime CLOCK_MONOTONIC);
use Fcntl;
use File::Map qw(map_handle unmap);

use Try::Tiny;
use bmwqemu;
Expand All @@ -33,15 +35,18 @@ sub screen ($self, @) {
return $self;
}

sub _stop_process ($self, $name) {
return undef unless my $pipe = delete $self->{$name};
my $pid = delete $self->{"${name}pid"};
kill(TERM => $pid);
close($pipe);
return waitpid($pid, 0);
}

sub disable_video ($self) {
my $ret = 0;
if ($self->{ffmpeg}) {
kill(TERM => $self->{ffmpegpid});
close($self->{ffmpeg});
$self->{ffmpeg} = undef;
$ret = waitpid($self->{ffmpegpid}, 0);
$self->{ffmpegpid} = undef;
}
$ret ||= $self->_stop_process('ffmpeg');
$ret ||= $self->_stop_process('ustreamer');
return $ret;
}

Expand Down Expand Up @@ -94,7 +99,7 @@ sub connect_remote ($self, $args) {
bmwqemu::diag "DV timings not supported";
}
} else {
# applies to v4l only
# applies to v4l via ffmpeg only
$self->{dv_timings_supported} = 0;
}

Expand All @@ -110,6 +115,15 @@ sub _get_ffmpeg_cmd ($self, $url) {
return \@cmd;
}

sub _get_ustreamer_cmd ($self, $url, $sink_name) {
return [
'ustreamer', '--device', $url, '-f', '5',
'-c', 'NOOP', # do not produce JPEG stream
'--raw-sink', $sink_name, '--raw-sink-rm', # raw memsink
'--dv-timings', # enable using DV timings (getting resolution, and reacting to changes)
];
}

sub connect_remote_video ($self, $url) {
if ($self->{dv_timings_supported}) {
if (!_v4l2_ctl($url, '--set-dv-bt-timings query')) {
Expand All @@ -120,15 +134,34 @@ sub connect_remote_video ($self, $url) {
$self->{dv_timings} = _v4l2_ctl($url, '--get-dv-timings');
}

my $cmd = $self->_get_ffmpeg_cmd($url);
my $ffmpeg;
$self->{ffmpegpid} = open($ffmpeg, '-|', @$cmd)
or die "Failed to start ffmpeg for video stream at $url";
# make the pipe size large enough to hold full frame and a bit
my $frame_size = $bmwqemu::vars{VIDEO_STREAM_PIPE_BUFFER_SIZE} // DEFAULT_VIDEO_STREAM_PIPE_BUFFER_SIZE;
fcntl($ffmpeg, Fcntl::F_SETPIPE_SZ, $frame_size);
$self->{ffmpeg} = $ffmpeg;
$ffmpeg->blocking(0);
if ($url =~ m^ustreamer://^) {
my $dev = ($url =~ m^ustreamer://(.*)^)[0];
my $sink_name = "raw-sink$dev";
$sink_name =~ s^/^-^g;
my $cmd = $self->_get_ustreamer_cmd($dev, $sink_name);
my $ffmpeg;
$self->{ustreamerpid} = open($ffmpeg, '-|', @$cmd)
or die "Failed to start ustreamer for video stream at $url";
$self->{ustreamer_pipe} = $ffmpeg;
my $timeout = 100;
while ($timeout && !-f "/dev/shm/$sink_name") {
sleep(0.1); # uncoverable statement
$timeout -= 1; # uncoverable statement
}
die "ustreamer startup timeout" if $timeout <= 0;
open($self->{ustreamer}, "+<", "/dev/shm/$sink_name")
or die "Failed to open ustreamer memsink";
} else {
my $cmd = $self->_get_ffmpeg_cmd($url);
my $ffmpeg;
$self->{ffmpegpid} = open($ffmpeg, '-|', @$cmd)
or die "Failed to start ffmpeg for video stream at $url";
# make the pipe size large enough to hold full frame and a bit
my $frame_size = $bmwqemu::vars{VIDEO_STREAM_PIPE_BUFFER_SIZE} // DEFAULT_VIDEO_STREAM_PIPE_BUFFER_SIZE;
fcntl($ffmpeg, Fcntl::F_SETPIPE_SZ, $frame_size);
$self->{ffmpeg} = $ffmpeg;
$ffmpeg->blocking(0);
}

$self->{_last_update_received} = time;

Expand All @@ -150,7 +183,7 @@ sub connect_remote_input ($self, $cmd) {
}


sub _receive_frame ($self) {
sub _receive_frame_ffmpeg ($self) {
my $ffmpeg = $self->{ffmpeg};
$ffmpeg or die 'ffmpeg is not running. Probably your backend instance could not start or died.';
$ffmpeg->blocking(0);
Expand Down Expand Up @@ -181,6 +214,85 @@ sub _receive_frame ($self) {
return $img;
}

sub _receive_frame_ustreamer ($self) {
die 'ustreamer is not running. Probably your backend instance could not start or died.'
unless my $ustreamer = $self->{ustreamer};

flock($self->{ustreamer}, Fcntl::LOCK_EX);
my $ustreamer_map;
map_handle($ustreamer_map, $ustreamer, "+<");
{
my $unlock = scope_guard sub {
unmap($ustreamer_map);
flock($ustreamer, Fcntl::LOCK_UN);
};

# us_memsink_shared_s struct defined in https://github.com/pikvm/ustreamer/blob/master/src/libs/memsinksh.h
# #define US_MEMSINK_MAGIC ((uint64_t)0xCAFEBABECAFEBABE)
# #define US_MEMSINK_VERSION ((uint32_t)4)
# typedef struct {
# uint64_t magic;
# uint32_t version;
# // pad
# uint64_t id;
#
# size_t used;
# unsigned width;
# unsigned height;
# unsigned format;
# unsigned stride;
# bool online;
# bool key;
# // pad
# unsigned gop;
# // 56
# long double grab_ts;
# long double encode_begin_ts;
# long double encode_end_ts;
# // 112
# long double last_client_ts;
# bool key_requested;
#
# // 192
# uint8_t data[US_MEMSINK_MAX_DATA];
# } us_memsink_shared_s;

my ($magic, $version, $id, $used, $width, $height, $format, $stride, $online, $key, $gop) =
unpack("QLx4QQIIa4ICCxxI", $ustreamer_map);
# This is US_MEMSINK_MAGIC, but perl considers hex literals over 32bits non-portable
if ($magic != 14627333968358193854) {
bmwqemu::diag "Invalid ustreamer magic: $magic";
return undef;
}
die "Unsupported ustreamer version '$version' (only version 4 supported)" if $version != 4;

# tell ustreamer we are reading, otherwise it won't write new frames
my $clock = clock_gettime(CLOCK_MONOTONIC);
substr($ustreamer_map, 112, 16) = pack("D", $clock);
# no new frame
return undef if $self->{ustreamer_last_id} && $id == $self->{ustreamer_last_id};
$self->{ustreamer_last_id} = $id;
# empty frame
return undef unless $used;

my $img;
if ($format eq 'JPEG') {
# tinycv::from_ppm in fact handles a bunch of formats, including JPEG
$img = tinycv::from_ppm(substr($ustreamer_map, 129, $used));
} elsif ($format eq 'UYVY') {
$img = tinycv::new($width, $height);
$img->map_raw_data_uyvy(substr($ustreamer_map, 129, $used));
} else {
die "Unsupported video format '$format'"; # uncoverable statement
}
$self->{_framebuffer} = $img;
$self->{width} = $width;
$self->{height} = $height;
$self->{_last_update_received} = time;
return $img;
}
}

sub update_framebuffer ($self) {
if ($self->{dv_timings_supported}) {
# periodically check if DV timings needs update due to resolution change
Expand All @@ -202,13 +314,21 @@ sub update_framebuffer ($self) {
}

# no video connected, don't read anything
return 0 unless $self->{ffmpeg};
return 0 unless $self->{ffmpeg} or $self->{ustreamer};

my $have_recieved_update = 0;
while ($self->_receive_frame()) {
$have_recieved_update = 1;
my $have_received_update = 0;
if ($self->{ffmpeg}) {
while ($self->_receive_frame_ffmpeg()) {
$have_received_update = 1;
}
} elsif ($self->{ustreamer}) {
# shared-memory interface "discards" older frames implicitly,
# no need to loop
if ($self->_receive_frame_ustreamer()) {
$have_received_update = 1;
}
}
return $have_recieved_update;
return $have_received_update;
}

sub current_screen ($self) {
Expand Down
Loading

0 comments on commit 5eb8a49

Please sign in to comment.