diff --git a/backend/generalhw.pm b/backend/generalhw.pm index 7804dcab466..4d5d9588d91 100644 --- a/backend/generalhw.pm +++ b/backend/generalhw.pm @@ -89,7 +89,8 @@ sub relogin_vnc ($self) { port => $bmwqemu::vars{GENERAL_HW_VNC_PORT} // 5900, password => $bmwqemu::vars{GENERAL_HW_VNC_PASSWORD}, depth => $bmwqemu::vars{GENERAL_HW_VNC_DEPTH} // 16, - connect_timeout => 50 + connect_timeout => 50, + jpeg => $bmwqemu::vars{GENERAL_HW_VNC_JPEG} // 0, }); $vnc->backend($self); $self->select_console({testapi_console => 'sut'}); diff --git a/consoles/VNC.pm b/consoles/VNC.pm index 19a43c569eb..3b010f0665c 100644 --- a/consoles/VNC.pm +++ b/consoles/VNC.pm @@ -19,7 +19,7 @@ has [qw(description hostname port username password socket name width height dep no_endian_conversion _pixinfo _colourmap _framebuffer _rfb_version screen_on _bpp _true_colour _do_endian_conversion absolute ikvm keymap _last_update_received _last_update_requested check_vnc_stalls _vnc_stalled vncinfo old_ikvm dell - vmware_vnc_over_ws_url original_hostname)]; + vmware_vnc_over_ws_url original_hostname jpeg)]; our $VERSION = '0.40'; @@ -83,17 +83,27 @@ my %supported_depths = ( }, ); +# Entries in order of preference my @encodings = ( - # These ones are defined in rfbproto.pdf + { + num => 16, + name => 'ZRLE', + supported => 1, + }, { num => 0, name => 'Raw', supported => 1, }, { - num => 16, - name => 'ZRLE', + num => 7, + name => 'Tight', + supported => 1, + }, + { + num => -24, + name => 'JPEG quality', supported => 1, }, { @@ -101,6 +111,11 @@ my @encodings = ( name => 'DesktopSize', supported => 1, }, + { + num => -224, + name => 'VNC_ENCODING_LAST_RECT', + supported => 1, + }, { num => -257, name => 'VNC_ENCODING_POINTER_TYPE_CHANGE', @@ -111,11 +126,6 @@ my @encodings = ( name => 'VNC_ENCODING_LED_STATE', supported => 1, }, - { - num => -224, - name => 'VNC_ENCODING_LAST_RECT', - supported => 1, - }, ); sub login ($self, $connect_timeout = undef, $timeout = undef) { @@ -428,14 +438,16 @@ sub _server_initialization ($self) { my @encs = grep { $_->{supported} } @encodings; - # Prefer the higher-numbered encodings - @encs = reverse sort { $a->{num} <=> $b->{num} } @encs; - if ($self->dell) { # idrac's ZRLE implementation even kills tigervnc, they duplicate # frames under certain conditions. Raw works ok @encs = grep { $_->{name} ne 'ZRLE' } @encs; } + if (!$self->jpeg) { + # according to the spec, RAW encoding is least preferred, so don't let + # loosy Tight JPEG be used over RAW unless explicitly requested + @encs = grep { $_->{name} ne 'Tight' and $_->{name} ne 'JPEG quality' } @encs; + } $socket->print( pack( 'CCn', @@ -840,6 +852,9 @@ sub _receive_update ($self) { $socket->read(my $data, $w * $h * $self->_bpp / 8) || die 'unexpected end of data'; $image->map_raw_data($data, $x, $y, $w, $h, $self->vncinfo); } + elsif ($encoding_type == 7) { # Tight + $self->_receive_tight_encoding($x, $y, $w, $h); + } elsif ($encoding_type == 16) { # ZRLE $self->_receive_zrle_encoding($x, $y, $w, $h); } @@ -921,6 +936,63 @@ sub _receive_zrle_encoding ($self, $x, $y, $w, $h) { return $res; } +# wrapper to make testing easier +sub _read_socket ($socket, $data, $data_len, $offset) { return read($socket, $data, $data_len, $offset); } # uncoverable statement + +sub _receive_tight_encoding ($self, $x, $y, $w, $h) { + my $socket = $self->socket; + my $image = $self->_framebuffer; + + $socket->read(my $data, 1) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control'); + my ($compression_control) = unpack('C', $data); + # FillCompression + if (($compression_control & 0xF0) == 0x80) { + my $data; + # special case for TPIXEL, otherwise identical to PIXEL + if ($self->_true_colour and $self->_bpp == 32 and $self->depth == 24) { + $socket->read($data, 3) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control'); + $data = pack('CCCx', unpack('CCC', $data)); + } else { + $socket->read($data, $self->_bpp / 8) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control'); + } + $image->fill_pixel($data, $self->vncinfo, $x, $y, $w, $h); + return; + } + # Only Fill (above) and JPEG for now; "Basic" unsupported + die "Unsupported compression $compression_control" if ($compression_control & 0xF0) != 0x90; + + $socket->read($data, 1) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len'); + my ($data_len) = unpack('C', $data); + if ($data_len & 0x80) { + $socket->read($data, 1) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len'); + my ($data_len2) = unpack('C', $data); + $data_len = ($data_len & 0x7f) | ($data_len2 & 0x7f) << 7; + if ($data_len2 & 0x80) { + $socket->read($data, 1) + or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len'); + my ($data_len2) = unpack('C', $data); + $data_len |= $data_len2 << 14; + } + } + + my $read_len = 0; + while ($read_len < $data_len) { + my $len = _read_socket($socket, $data, $data_len - $read_len, $read_len); + OpenQA::Exception::VNCProtocolError->throw(error => "short read for jpeg data $read_len - $data_len") unless $len; + $read_len += $len; + } + my $rect = tinycv::from_ppm($data); + OpenQA::Exception::VNCProtocolError->throw(error => "Invalid width/height of the rectangle (${w}x${h} != " . $rect->xres . "x" . $rect->yres . ")") + unless $w == $rect->xres and $h == $rect->yres; + $image->blend($rect, $x, $y); + $self->_framebuffer($image); +} + sub _receive_ikvm_encoding ($self, $encoding_type, $x, $y, $w, $h) { my $socket = $self->socket; my $image = $self->_framebuffer; diff --git a/consoles/video_stream.pm b/consoles/video_stream.pm index fd530344cb8..985e9490422 100644 --- a/consoles/video_stream.pm +++ b/consoles/video_stream.pm @@ -7,10 +7,12 @@ package consoles::video_stream; use Mojo::Base 'consoles::video_base', -signatures; use Mojo::UserAgent; use Mojo::URL; +use Mojo::Util 'scope_guard'; use List::Util 'max'; -use Time::HiRes qw(usleep); +use Time::HiRes qw(usleep clock_gettime CLOCK_MONOTONIC); use Fcntl; +use File::Map qw(map_handle unmap); use Try::Tiny; use bmwqemu; @@ -33,15 +35,18 @@ sub screen ($self, @) { return $self; } +sub _stop_process ($self, $name) { + return undef unless my $pipe = delete $self->{$name}; + my $pid = delete $self->{"${name}pid"}; + kill(TERM => $pid); + close($pipe); + return waitpid($pid, 0); +} + sub disable_video ($self) { my $ret = 0; - if ($self->{ffmpeg}) { - kill(TERM => $self->{ffmpegpid}); - close($self->{ffmpeg}); - $self->{ffmpeg} = undef; - $ret = waitpid($self->{ffmpegpid}, 0); - $self->{ffmpegpid} = undef; - } + $ret ||= $self->_stop_process('ffmpeg'); + $ret ||= $self->_stop_process('ustreamer'); return $ret; } @@ -94,7 +99,7 @@ sub connect_remote ($self, $args) { bmwqemu::diag "DV timings not supported"; } } else { - # applies to v4l only + # applies to v4l via ffmpeg only $self->{dv_timings_supported} = 0; } @@ -110,6 +115,15 @@ sub _get_ffmpeg_cmd ($self, $url) { return \@cmd; } +sub _get_ustreamer_cmd ($self, $url, $sink_name) { + return [ + 'ustreamer', '--device', $url, '-f', '5', + '-c', 'NOOP', # do not produce JPEG stream + '--raw-sink', $sink_name, '--raw-sink-rm', # raw memsink + '--dv-timings', # enable using DV timings (getting resolution, and reacting to changes) + ]; +} + sub connect_remote_video ($self, $url) { if ($self->{dv_timings_supported}) { if (!_v4l2_ctl($url, '--set-dv-bt-timings query')) { @@ -120,15 +134,34 @@ sub connect_remote_video ($self, $url) { $self->{dv_timings} = _v4l2_ctl($url, '--get-dv-timings'); } - my $cmd = $self->_get_ffmpeg_cmd($url); - my $ffmpeg; - $self->{ffmpegpid} = open($ffmpeg, '-|', @$cmd) - or die "Failed to start ffmpeg for video stream at $url"; - # make the pipe size large enough to hold full frame and a bit - my $frame_size = $bmwqemu::vars{VIDEO_STREAM_PIPE_BUFFER_SIZE} // DEFAULT_VIDEO_STREAM_PIPE_BUFFER_SIZE; - fcntl($ffmpeg, Fcntl::F_SETPIPE_SZ, $frame_size); - $self->{ffmpeg} = $ffmpeg; - $ffmpeg->blocking(0); + if ($url =~ m^ustreamer://^) { + my $dev = ($url =~ m^ustreamer://(.*)^)[0]; + my $sink_name = "raw-sink$dev"; + $sink_name =~ s^/^-^g; + my $cmd = $self->_get_ustreamer_cmd($dev, $sink_name); + my $ffmpeg; + $self->{ustreamerpid} = open($ffmpeg, '-|', @$cmd) + or die "Failed to start ustreamer for video stream at $url"; + $self->{ustreamer_pipe} = $ffmpeg; + my $timeout = 100; + while ($timeout && !-f "/dev/shm/$sink_name") { + sleep(0.1); # uncoverable statement + $timeout -= 1; # uncoverable statement + } + die "ustreamer startup timeout" if $timeout <= 0; + open($self->{ustreamer}, "+<", "/dev/shm/$sink_name") + or die "Failed to open ustreamer memsink"; + } else { + my $cmd = $self->_get_ffmpeg_cmd($url); + my $ffmpeg; + $self->{ffmpegpid} = open($ffmpeg, '-|', @$cmd) + or die "Failed to start ffmpeg for video stream at $url"; + # make the pipe size large enough to hold full frame and a bit + my $frame_size = $bmwqemu::vars{VIDEO_STREAM_PIPE_BUFFER_SIZE} // DEFAULT_VIDEO_STREAM_PIPE_BUFFER_SIZE; + fcntl($ffmpeg, Fcntl::F_SETPIPE_SZ, $frame_size); + $self->{ffmpeg} = $ffmpeg; + $ffmpeg->blocking(0); + } $self->{_last_update_received} = time; @@ -150,7 +183,7 @@ sub connect_remote_input ($self, $cmd) { } -sub _receive_frame ($self) { +sub _receive_frame_ffmpeg ($self) { my $ffmpeg = $self->{ffmpeg}; $ffmpeg or die 'ffmpeg is not running. Probably your backend instance could not start or died.'; $ffmpeg->blocking(0); @@ -181,6 +214,85 @@ sub _receive_frame ($self) { return $img; } +sub _receive_frame_ustreamer ($self) { + die 'ustreamer is not running. Probably your backend instance could not start or died.' + unless my $ustreamer = $self->{ustreamer}; + + flock($self->{ustreamer}, Fcntl::LOCK_EX); + my $ustreamer_map; + map_handle($ustreamer_map, $ustreamer, "+<"); + { + my $unlock = scope_guard sub { + unmap($ustreamer_map); + flock($ustreamer, Fcntl::LOCK_UN); + }; + + # us_memsink_shared_s struct defined in https://github.com/pikvm/ustreamer/blob/master/src/libs/memsinksh.h + # #define US_MEMSINK_MAGIC ((uint64_t)0xCAFEBABECAFEBABE) + # #define US_MEMSINK_VERSION ((uint32_t)4) + # typedef struct { + # uint64_t magic; + # uint32_t version; + # // pad + # uint64_t id; + # + # size_t used; + # unsigned width; + # unsigned height; + # unsigned format; + # unsigned stride; + # bool online; + # bool key; + # // pad + # unsigned gop; + # // 56 + # long double grab_ts; + # long double encode_begin_ts; + # long double encode_end_ts; + # // 112 + # long double last_client_ts; + # bool key_requested; + # + # // 192 + # uint8_t data[US_MEMSINK_MAX_DATA]; + # } us_memsink_shared_s; + + my ($magic, $version, $id, $used, $width, $height, $format, $stride, $online, $key, $gop) = + unpack("QLx4QQIIa4ICCxxI", $ustreamer_map); + # This is US_MEMSINK_MAGIC, but perl considers hex literals over 32bits non-portable + if ($magic != 14627333968358193854) { + bmwqemu::diag "Invalid ustreamer magic: $magic"; + return undef; + } + die "Unsupported ustreamer version '$version' (only version 4 supported)" if $version != 4; + + # tell ustreamer we are reading, otherwise it won't write new frames + my $clock = clock_gettime(CLOCK_MONOTONIC); + substr($ustreamer_map, 112, 16) = pack("D", $clock); + # no new frame + return undef if $self->{ustreamer_last_id} && $id == $self->{ustreamer_last_id}; + $self->{ustreamer_last_id} = $id; + # empty frame + return undef unless $used; + + my $img; + if ($format eq 'JPEG') { + # tinycv::from_ppm in fact handles a bunch of formats, including JPEG + $img = tinycv::from_ppm(substr($ustreamer_map, 129, $used)); + } elsif ($format eq 'UYVY') { + $img = tinycv::new($width, $height); + $img->map_raw_data_uyvy(substr($ustreamer_map, 129, $used)); + } else { + die "Unsupported video format '$format'"; # uncoverable statement + } + $self->{_framebuffer} = $img; + $self->{width} = $width; + $self->{height} = $height; + $self->{_last_update_received} = time; + return $img; + } +} + sub update_framebuffer ($self) { if ($self->{dv_timings_supported}) { # periodically check if DV timings needs update due to resolution change @@ -202,13 +314,21 @@ sub update_framebuffer ($self) { } # no video connected, don't read anything - return 0 unless $self->{ffmpeg}; + return 0 unless $self->{ffmpeg} or $self->{ustreamer}; - my $have_recieved_update = 0; - while ($self->_receive_frame()) { - $have_recieved_update = 1; + my $have_received_update = 0; + if ($self->{ffmpeg}) { + while ($self->_receive_frame_ffmpeg()) { + $have_received_update = 1; + } + } elsif ($self->{ustreamer}) { + # shared-memory interface "discards" older frames implicitly, + # no need to loop + if ($self->_receive_frame_ustreamer()) { + $have_received_update = 1; + } } - return $have_recieved_update; + return $have_received_update; } sub current_screen ($self) { diff --git a/container/os-autoinst_dev/Dockerfile b/container/os-autoinst_dev/Dockerfile index c3a3e2f9d1a..63bd8efbdf5 100644 --- a/container/os-autoinst_dev/Dockerfile +++ b/container/os-autoinst_dev/Dockerfile @@ -76,6 +76,7 @@ RUN zypper in -y -C \ 'perl(Fcntl)' \ 'perl(File::Basename)' \ 'perl(File::Find)' \ + 'perl(File::Map)' \ 'perl(File::Path)' \ 'perl(File::Temp)' \ 'perl(File::Touch)' \ diff --git a/cpanfile b/cpanfile index 0d3b5834657..353a772853a 100644 --- a/cpanfile +++ b/cpanfile @@ -22,6 +22,7 @@ requires 'ExtUtils::testlib'; requires 'Fcntl'; requires 'File::Basename'; requires 'File::Find'; +requires 'File::Map'; requires 'File::Path'; requires 'File::Temp'; requires 'File::Touch'; diff --git a/dependencies.yaml b/dependencies.yaml index 7da9b238e0f..830752eb532 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -138,6 +138,7 @@ main_requires: perl(File::Basename): perl(File::chdir): perl(File::Find): + perl(File::Map): perl(File::Path): perl(File::Temp): perl(File::Touch): diff --git a/dist/rpm/os-autoinst.spec b/dist/rpm/os-autoinst.spec index 8febe589454..e848e66448d 100644 --- a/dist/rpm/os-autoinst.spec +++ b/dist/rpm/os-autoinst.spec @@ -36,7 +36,7 @@ Source0: %{name}-%{version}.tar.xz # The following line is generated from dependencies.yaml %define build_requires %build_base_requires cmake ninja # The following line is generated from dependencies.yaml -%define main_requires git-core perl(B::Deparse) perl(Carp) perl(Carp::Always) perl(Config) perl(Cpanel::JSON::XS) perl(Crypt::DES) perl(Cwd) perl(Data::Dumper) perl(Digest::MD5) perl(DynaLoader) perl(English) perl(Errno) perl(Exception::Class) perl(Exporter) perl(ExtUtils::testlib) perl(Fcntl) perl(File::Basename) perl(File::Find) perl(File::Path) perl(File::Temp) perl(File::Touch) perl(File::Which) perl(File::chdir) perl(IO::Handle) perl(IO::Scalar) perl(IO::Select) perl(IO::Socket) perl(IO::Socket::INET) perl(IO::Socket::UNIX) perl(IPC::Open3) perl(IPC::Run::Debug) perl(IPC::System::Simple) perl(JSON::Validator) perl(List::MoreUtils) perl(List::Util) perl(Mojo::IOLoop::ReadWriteProcess) >= 0.26 perl(Mojo::JSON) perl(Mojo::Log) perl(Mojo::URL) perl(Mojo::UserAgent) perl(Mojolicious) >= 9.340.0 perl(Mojolicious::Lite) perl(Net::DBus) perl(Net::Domain) perl(Net::IP) perl(Net::SNMP) perl(Net::SSH2) perl(POSIX) perl(Scalar::Util) perl(Socket) perl(Socket::MsgHdr) perl(Term::ANSIColor) perl(Thread::Queue) perl(Time::HiRes) perl(Time::Moment) perl(Time::Seconds) perl(Try::Tiny) perl(XML::LibXML) perl(XML::SemanticDiff) perl(YAML::PP) perl(YAML::XS) perl(autodie) perl(base) perl(constant) perl(integer) perl(strict) perl(version) perl(warnings) perl-base rsync sshpass +%define main_requires git-core perl(B::Deparse) perl(Carp) perl(Carp::Always) perl(Config) perl(Cpanel::JSON::XS) perl(Crypt::DES) perl(Cwd) perl(Data::Dumper) perl(Digest::MD5) perl(DynaLoader) perl(English) perl(Errno) perl(Exception::Class) perl(Exporter) perl(ExtUtils::testlib) perl(Fcntl) perl(File::Basename) perl(File::Find) perl(File::Map) perl(File::Path) perl(File::Temp) perl(File::Touch) perl(File::Which) perl(File::chdir) perl(IO::Handle) perl(IO::Scalar) perl(IO::Select) perl(IO::Socket) perl(IO::Socket::INET) perl(IO::Socket::UNIX) perl(IPC::Open3) perl(IPC::Run::Debug) perl(IPC::System::Simple) perl(JSON::Validator) perl(List::MoreUtils) perl(List::Util) perl(Mojo::IOLoop::ReadWriteProcess) >= 0.26 perl(Mojo::JSON) perl(Mojo::Log) perl(Mojo::URL) perl(Mojo::UserAgent) perl(Mojolicious) >= 9.340.0 perl(Mojolicious::Lite) perl(Net::DBus) perl(Net::Domain) perl(Net::IP) perl(Net::SNMP) perl(Net::SSH2) perl(POSIX) perl(Scalar::Util) perl(Socket) perl(Socket::MsgHdr) perl(Term::ANSIColor) perl(Thread::Queue) perl(Time::HiRes) perl(Time::Moment) perl(Time::Seconds) perl(Try::Tiny) perl(XML::LibXML) perl(XML::SemanticDiff) perl(YAML::PP) perl(YAML::XS) perl(autodie) perl(base) perl(constant) perl(integer) perl(strict) perl(version) perl(warnings) perl-base rsync sshpass # all requirements needed by the tests, do not require on this in the package # itself or any sub-packages # SLE is missing spell check requirements diff --git a/doc/backend_vars.asciidoc b/doc/backend_vars.asciidoc index 601757fb6bf..00084236a8e 100644 --- a/doc/backend_vars.asciidoc +++ b/doc/backend_vars.asciidoc @@ -309,8 +309,9 @@ GENERAL_HW_VNC_IP;string;;Hostname of the gadget's network. If not set, SSH cons GENERAL_HW_VNC_PASSWORD;string;;Password for VNC server GENERAL_HW_VNC_PORT;integer;5900;VNC Port number GENERAL_HW_VNC_DEPTH;integer;16;Color depth for VNC server +GENERAL_HW_VNC_JPEG;integer;0;Advertise support for Tight JPEG encoding GENERAL_HW_NO_SERIAL;boolean;;Don't use serial -GENERAL_HW_VIDEO_STREAM_URL;string;;Video stream URL (in ffmpeg's syntax) to receive, for example 'udp://@:5004' or '/dev/video0'. +GENERAL_HW_VIDEO_STREAM_URL;string;;Video stream URL (in ffmpeg's syntax) to receive, for example 'udp://@:5004' or '/dev/video0'. Using 'ustreamer:///dev/videoN' will use ustreamer from PiKVM instead of ffmpeg to read '/dev/videoN'. Ustreamer support requires pack("D") working, which rules out openSUSE 15.5's perl. VIDEO_STREAM_PIPE_BUFFER_SIZE;integer;1680*1050*3+20;Buffer containing at least a single PPM frame for video capturing GENERAL_HW_KEYBOARD_URL;string;;URL to keyboard emulation device. eg. 'http://1.2.3.4/cmd' - see https://github.com/os-autoinst/os-autoinst-distri-opensuse/tree/master/data/generalhw_scripts/rpi_pico_w_keyboard[rpi_pico_w_keyboard] GENERAL_HW_CMD_DIR;string;;Directory with allowed CMD scripts. Note: This variable should be set in the workers.ini file, otherwise it will be ignored by openQA. diff --git a/ppmclibs/tinycv.h b/ppmclibs/tinycv.h index 5214777289c..e22fb3265f6 100644 --- a/ppmclibs/tinycv.h +++ b/ppmclibs/tinycv.h @@ -48,6 +48,8 @@ VNCInfo *image_vncinfo(bool do_endian_conversion, std::tuple image_get_vnc_color(VNCInfo* info, unsigned int index); void image_set_vnc_color(VNCInfo *info, unsigned int index, unsigned int red, unsigned int green, unsigned int blue); +void image_fill_pixel(Image* a, const unsigned char *data, VNCInfo* info, + long x, long y, long width, long height); // this is for VNC support - RAW encoding void image_map_raw_data(Image *a, const unsigned char *data, unsigned int x, unsigned int y, unsigned int width, unsigned int height, VNCInfo *info); @@ -62,5 +64,8 @@ long image_map_raw_data_zrle(Image* a, long x, long y, long w, long h, unsigned char *data, size_t len); +// raw data from v4l2 +void image_map_raw_data_uyvy(Image *a, const unsigned char *data); + // copy the s image into a at x,y void image_blend_image(Image *a, Image *s, long x, long y); diff --git a/ppmclibs/tinycv.xs b/ppmclibs/tinycv.xs index cc2b14a810c..0fb94533468 100644 --- a/ppmclibs/tinycv.xs +++ b/ppmclibs/tinycv.xs @@ -208,6 +208,14 @@ long map_raw_data_zrle(tinycv::Image self, long x, long y, long w, long h, tinyc OUTPUT: RETVAL +void fill_pixel(tinycv::Image self, unsigned char *data, tinycv::VNCInfo info, long x, long y, long w, long h) + CODE: + image_fill_pixel(self, data, info, x, y, w, h); + +void map_raw_data_uyvy(tinycv::Image self, unsigned char *data) + CODE: + image_map_raw_data_uyvy(self, data); + void blend(tinycv::Image self, tinycv::Image source, long x, long y) CODE: image_blend_image(self, source, x, y); diff --git a/ppmclibs/tinycv_impl.cc b/ppmclibs/tinycv_impl.cc index 93f40571e84..cb6c25866e2 100644 --- a/ppmclibs/tinycv_impl.cc +++ b/ppmclibs/tinycv_impl.cc @@ -617,6 +617,28 @@ VNCInfo* image_vncinfo(bool do_endian_conversion, bool true_colour, blue_shift); } +/* + * Fill given rectangle with pixel value read from 'data' and interpreted + * according to 'info'. Number of bytes read from 'data' depends on 'info'. + */ +void image_fill_pixel(Image* a, const unsigned char *data, VNCInfo* info, + long x, long y, long width, long height) +{ + size_t offset = 0; + Vec3b pixel = info->read_pixel(data, offset); + + // avoid an exception + if (x < 0 || y < 0 || y + height > a->img.rows || x + width > a->img.cols) { + std::cerr << "ERROR - fill_pixel: out of range\n" + << std::endl; + return; + } + + for (auto i = static_cast(y); i < y + height; i++) + for (auto j = static_cast(x); j < x + width; j++) + a->img.at(i, j) = pixel; +} + // implemented in tinycv_ast2100 void decode_ast2100(Mat* img, const unsigned char* data, size_t len); @@ -645,6 +667,48 @@ void image_map_raw_data_rgb555(Image* a, const unsigned char* data) } } +void image_map_raw_data_uyvy(Image* a, const unsigned char* data) +{ + for (int y = 0; y < a->img.rows; y++) { + for (int x = 0; x < a->img.cols; x += 2) { + int offset = (y * a->img.cols + x) * 2; + int u = data[offset + 0]; + int y1 = data[offset + 1]; + int v = data[offset + 2]; + int y2 = data[offset + 3]; + + y1 -= 16; + y2 -= 16; + int cb = u - 128; + int cr = v - 128; + + int r1 = (298 * y1 + 409 * cr + 128) >> 8; + int g1 = (298 * y1 - 100 * cb - 208 * cr + 128) >> 8; + int b1 = (298 * y1 + 516 * cb + 128) >> 8; + + int r2 = (298 * y2 + 409 * cr + 128) >> 8; + int g2 = (298 * y2 - 100 * cb - 208 * cr + 128) >> 8; + int b2 = (298 * y2 + 516 * cb + 128) >> 8; + + // Clamp values to the valid range [0, 255] + r1 = (r1 < 0) ? 0 : ((r1 > 255) ? 255 : r1); + g1 = (g1 < 0) ? 0 : ((g1 > 255) ? 255 : g1); + b1 = (b1 < 0) ? 0 : ((b1 > 255) ? 255 : b1); + + r2 = (r2 < 0) ? 0 : ((r2 > 255) ? 255 : r2); + g2 = (g2 < 0) ? 0 : ((g2 > 255) ? 255 : g2); + b2 = (b2 < 0) ? 0 : ((b2 > 255) ? 255 : b2); + + a->img.at(y, x)[0] = b1; + a->img.at(y, x)[1] = g1; + a->img.at(y, x)[2] = r1; + a->img.at(y, x + 1)[0] = b2; + a->img.at(y, x + 1)[1] = g2; + a->img.at(y, x + 1)[2] = r2; + } + } +} + static uint16_t read_u16(const unsigned char* data, size_t& offset, bool do_endian_conversion) { diff --git a/t/26-video_stream.t b/t/26-video_stream.t index 064443692b7..f1c9042a4a1 100755 --- a/t/26-video_stream.t +++ b/t/26-video_stream.t @@ -9,6 +9,7 @@ use Mojo::UserAgent; use Mojo::Transaction::HTTP; use Test::Warnings qw(:all :report_warnings); use File::Basename; +use File::Copy; use FindBin '$Bin'; use lib "$Bin/../external/os-autoinst-common/lib"; use OpenQA::Test::TimeLimit '5'; @@ -33,6 +34,7 @@ $mock_console->redefine(_get_ffmpeg_cmd => sub ($self, $url) { my @cmd = ('cat', $mock_video_source); return \@cmd; }); +$mock_console->redefine(_get_ustreamer_cmd => ["true"]); my $mock_backend = Test::MockObject->new(); $mock_backend->{xres} = 1024; @@ -94,6 +96,19 @@ subtest 'connect stream' => sub { [('/dev/video0', '--get-dv-timings')], ], "calls to v4l2-ctl"; + @v4l2_ctl_calls = (); + copy($data_dir . "frame1.ppm", '/dev/shm/raw-sink-dev-video0'); + $console->connect_remote({url => 'ustreamer:///dev/video0'}); + is $console->{dv_timings_supported}, 0, "correctly skipping DV timing"; + is_deeply \@v4l2_ctl_calls, [], "calls to v4l2-ctl"; + + my $cmd = $mock_console->original('_get_ustreamer_cmd')->($console, '/dev/video0', 'raw-sink-dev-video0'); + is_deeply $cmd, [ + 'ustreamer', '--device', '/dev/video0', '-f', '5', + '-c', 'NOOP', + '--raw-sink', 'raw-sink-dev-video0', '--raw-sink-rm', + '--dv-timings'], "correct cmd built"; + }; subtest 'frames parsing' => sub { @@ -117,6 +132,40 @@ subtest 'frames parsing' => sub { $console->disable_video; }; +subtest 'frame parsing - ustreamer' => sub { + # ustreamer requires pack("D") support, not availabe in openSUSE Leap 15.5's Perl + eval { $_ = pack("D", 1.0); }; + plan skip_all => 'packing long double is not supported' if $@; + my ($img, $received_img); + + # ustreamer frame, invalid magic + copy($data_dir . "frame1.ppm", '/dev/shm/raw-sink-dev-video0'); + my $console = consoles::video_stream->new(undef, {url => 'ustreamer:///dev/video0'}); + $console->connect_remote({url => 'ustreamer:///dev/video0'}); + + my $received_update = $console->update_framebuffer(); + is $received_update, 0, "detected invalid data"; + $console->disable_video; + + # ustreamer frame, "no signal" message encoded as JPEG + copy($data_dir . "ustreamer-shared-no-signal", '/dev/shm/raw-sink-dev-video0'); + $console->connect_remote({url => 'ustreamer:///dev/video0'}); + + $img = tinycv::read($data_dir . "ustreamer-shared-no-signal.png"); + $received_img = $console->current_screen(); + is $received_img->similarity($img), 1_000_000, "received correct JPEG frame"; + $console->disable_video; + + # ustreamer frame, actual data, encoded as UYVY + copy($data_dir . "ustreamer-shared-full-frame", '/dev/shm/raw-sink-dev-video0'); + $console->connect_remote({url => 'ustreamer:///dev/video0'}); + + $img = tinycv::read($data_dir . "ustreamer-shared-full-frame.png"); + $received_img = $console->current_screen(); + is $received_img->similarity($img), 1_000_000, "received correct UYVY frame"; + $console->disable_video; +}; + subtest 'v4l2 resolution' => sub { $mock_video_source = $data_dir . "frame1.ppm"; my $console = consoles::video_stream->new(undef, {url => '/dev/video0'}); diff --git a/t/27-consoles-vnc.t b/t/27-consoles-vnc.t index ade49b79ba0..4053715b786 100755 --- a/t/27-consoles-vnc.t +++ b/t/27-consoles-vnc.t @@ -7,6 +7,7 @@ use Test::Most; use Mojo::Base -strict, -signatures; use utf8; +use Mojo::File qw(path); use Test::Warnings qw(:all :report_warnings); use Test::Exception; use Test::Output qw(combined_like); @@ -14,6 +15,7 @@ use Test::MockModule; use Test::MockObject; use Test::Mock::Time; use FindBin '$Bin'; +use File::Basename; use lib "$Bin/../external/os-autoinst-common/lib"; use OpenQA::Test::TimeLimit '5'; use consoles::VNC; @@ -32,7 +34,8 @@ my $inet_mock = Test::MockModule->new('IO::Socket::INET'); my $s = Test::MockObject->new->set_true(qw(sockopt fileno print connected close blocking)); sub _setup_rfb_magic () { $s->set_series('mocked_read', 'RFB 003.006', pack('N', 1)) } _setup_rfb_magic; -$s->mock(read => sub { $_[1] = $s->mocked_read; defined $_[1] }); +$s->mock(read => sub { $_[1] = $s->mocked_read; length $_[1] }); +$vnc_mock->redefine(_read_socket => sub { substr($_[1], $_[3], $_[2]) = $s->mocked_read; length $_[1] }); $s->mock($_ => sub { push @printed, $_[1] }) for qw(print write); $inet_mock->redefine(new => $s); $vnc_mock->noop('_server_initialization'); @@ -243,6 +246,54 @@ subtest 'update framebuffer' => sub { $s->set_series(mocked_read => $update_message, $one_rectangle, $ikvm_encoding, $ikvm_specific_data, $actual_image_data); $c->update_framebuffer; is scalar @printed, 1, 'no further image requested' or diag explain \@printed; + + my $of_type_tight_with_coordinates_12_42_4_4 = pack(nnnnNC => 12, 42, 4, 4, 7); + my $fill_compression = pack("C", 0x80); + subtest 'Tight encoding, FillCompression' => sub { + $c->_do_endian_conversion($machine_is_big_endian); # assume server is little-endian + $vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16); + $gray_pixel = pack(CCCC => 31, 37, 41, 0); # dark prime grey + $s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_12_42_4_4, $fill_compression, $gray_pixel); + $c->_framebuffer(undef)->width(1024)->height(512)->vncinfo($vncinfo); + ok $c->update_framebuffer, 'truthy return value for successful pixel update'; + ($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44); + is $blue, 41, 'pixel data updated in framebuffer (blue)'; + is $green, 37, 'pixel data updated in framebuffer (green)'; + is $red, 31, 'pixel data updated in framebuffer (red)'; + }; + + subtest 'Tight encoding, FillCompression, TPIXEL' => sub { + $c->_do_endian_conversion($machine_is_big_endian); # assume server is little-endian + $c->depth(24); + $c->_true_colour(1); + $vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16); + $gray_pixel = pack(CCCC => 31, 37, 41); # dark prime grey + $s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_12_42_4_4, $fill_compression, $gray_pixel); + $c->_framebuffer(undef)->width(1024)->height(512)->vncinfo($vncinfo); + ok $c->update_framebuffer, 'truthy return value for successful pixel update'; + ($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44); + is $blue, 41, 'pixel data updated in framebuffer (blue)'; + is $green, 37, 'pixel data updated in framebuffer (green)'; + is $red, 31, 'pixel data updated in framebuffer (red)'; + }; + + subtest 'Tight encoding, JpegCompression' => sub { + my $jpeg_data = path(dirname(__FILE__) . '/data/frame1.jpeg')->slurp; + my $of_type_tight_with_coordinates_0_0_1024_768 = pack(nnnnNC => 0, 0, 1024, 768, 7); + my $jpeg_compression = pack("C", 0x90); + my $data_len = length($jpeg_data); + my $data_len1 = pack("C", ($data_len & 0x7f) | 0x80); + my $data_len2 = pack("C", (($data_len >> 7) & 0x7f) | 0x80); + my $data_len3 = pack("C", ($data_len >> 14)); + + $s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_0_0_1024_768, $jpeg_compression, $data_len1, $data_len2, $data_len3, $jpeg_data); + $c->_framebuffer(undef)->width(1024)->height(768); + ok $c->update_framebuffer, 'truthy return value for successful pixel update'; + ($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44); + is $blue, 0x3e, 'pixel data updated in framebuffer (blue)'; + is $green, 0x84, 'pixel data updated in framebuffer (green)'; + is $red, 0x01, 'pixel data updated in framebuffer (red)'; + }; }; subtest 'read special messages/encodings' => sub { @@ -368,6 +419,25 @@ subtest 'server initialization' => sub { pack(N => -261), # VNC_ENCODING_LED_STATE ); is_deeply \@printed, \@expected, 'pixel format and encodings replied' or diag explain \@printed; + + # test with jpeg enabled + @printed = (); + $s->set_series(mocked_read => $server_init); + $c->jpeg(1); + $c->_server_initialization; + # expect params for 16-bit depth being replied as setpixelformat + @expected = ( + pack(CCCCCCCCnnnCCCCCC => 0, 0, 0, 0, @params, 0, 0, 0), # setpixelformat + pack(CCn => 2, 0, 7), # seven supported encodings (no ZRLE due to dell flag) + pack(N => 0000), # raw + pack(N => 0007), # Tight + pack(N => -24), # JPEG quality + pack(N => -223), # DesktopSize + pack(N => -224), # VNC_ENCODING_LAST_RECT + pack(N => -257), # VNC_ENCODING_POINTER_TYPE_CHANGE + pack(N => -261), # VNC_ENCODING_LED_STATE + ); + is_deeply \@printed, \@expected, 'pixel format and encodings replied' or diag explain \@printed; }; subtest 'login on real VNC server via vnctest, request and receive frame buffer' => sub { diff --git a/t/data/frame1.jpeg b/t/data/frame1.jpeg new file mode 100644 index 00000000000..29a68892255 Binary files /dev/null and b/t/data/frame1.jpeg differ diff --git a/t/data/ustreamer-shared-full-frame b/t/data/ustreamer-shared-full-frame new file mode 100644 index 00000000000..5ec154f0703 Binary files /dev/null and b/t/data/ustreamer-shared-full-frame differ diff --git a/t/data/ustreamer-shared-full-frame.png b/t/data/ustreamer-shared-full-frame.png new file mode 100644 index 00000000000..e2dcf29b514 Binary files /dev/null and b/t/data/ustreamer-shared-full-frame.png differ diff --git a/t/data/ustreamer-shared-no-signal b/t/data/ustreamer-shared-no-signal new file mode 100644 index 00000000000..325dcd59dfd Binary files /dev/null and b/t/data/ustreamer-shared-no-signal differ diff --git a/t/data/ustreamer-shared-no-signal.png b/t/data/ustreamer-shared-no-signal.png new file mode 100644 index 00000000000..7cf3e80284c Binary files /dev/null and b/t/data/ustreamer-shared-no-signal.png differ