Skip to content

Commit

Permalink
VNC: add support for Tight encoding
Browse files Browse the repository at this point in the history 
This extends supported VNC servers, especially it can be used with
the one from PiKVM (useful when openqa-worker is running on a different
host than PiKVM, otherwise ustreamer will give better results).

This implementation supports JPEG and Fill compression, which is enough
to serve PiKVM (only JPEG used) and Xvnc (both Fill and JPEG used).
BasicCompression is not implemented here.
To make lossless encodings preferred, place them manually at the
beginning of the list, instead of sorting by value. But also, disable it
by default, since by the spec it should be preferred over RAW (but not
over ZRLE). To enable it with generalhw backend, set
GENERAL_HW_VNC_JPEG=1.

Similar implementation (if not exactly the same) could be used for
implementing Tight PNG encoding (-260).
JPEG encoding (21) is a bit different, as there is no explicit size
(parser needs to read data until EOF marker), and it may re-use Huffman
tables.
Neither of the last two is needed for PiKVM, so this commit do not
implement them.
  • Loading branch information
@marmarek
marmarek committed Jan 10, 2024
1 parent 21ac3a9 commit 0195131
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 13 deletions.
3 changes: 2 additions & 1 deletion backend/generalhw.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ sub relogin_vnc ($self) {
port => $bmwqemu::vars{GENERAL_HW_VNC_PORT} // 5900,
password => $bmwqemu::vars{GENERAL_HW_VNC_PASSWORD},
depth => $bmwqemu::vars{GENERAL_HW_VNC_DEPTH} // 16,
connect_timeout => 50
connect_timeout => 50,
jpeg => $bmwqemu::vars{GENERAL_HW_VNC_JPEG} // 0,
});
$vnc->backend($self);
$self->select_console({testapi_console => 'sut'});
Expand Down
96 changes: 84 additions & 12 deletions consoles/VNC.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ has [qw(description hostname port username password socket name width height dep
no_endian_conversion _pixinfo _colourmap _framebuffer _rfb_version screen_on
_bpp _true_colour _do_endian_conversion absolute ikvm keymap _last_update_received
_last_update_requested check_vnc_stalls _vnc_stalled vncinfo old_ikvm dell
vmware_vnc_over_ws_url original_hostname)];
vmware_vnc_over_ws_url original_hostname jpeg)];

our $VERSION = '0.40';

Expand Down Expand Up @@ -83,24 +83,39 @@ my %supported_depths = (
},
);

# Entries in order of preference
my @encodings = (

# These ones are defined in rfbproto.pdf
{
num => 16,
name => 'ZRLE',
supported => 1,
},
{
num => 0,
name => 'Raw',
supported => 1,
},
{
num => 16,
name => 'ZRLE',
num => 7,
name => 'Tight',
supported => 1,
},
{
num => -24,
name => 'JPEG quality',
supported => 1,
},
{
num => -223,
name => 'DesktopSize',
supported => 1,
},
{
num => -224,
name => 'VNC_ENCODING_LAST_RECT',
supported => 1,
},
{
num => -257,
name => 'VNC_ENCODING_POINTER_TYPE_CHANGE',
Expand All @@ -111,11 +126,6 @@ my @encodings = (
name => 'VNC_ENCODING_LED_STATE',
supported => 1,
},
{
num => -224,
name => 'VNC_ENCODING_LAST_RECT',
supported => 1,
},
);

sub login ($self, $connect_timeout = undef, $timeout = undef) {
Expand Down Expand Up @@ -428,14 +438,16 @@ sub _server_initialization ($self) {

my @encs = grep { $_->{supported} } @encodings;

# Prefer the higher-numbered encodings
@encs = reverse sort { $a->{num} <=> $b->{num} } @encs;

if ($self->dell) {
# idrac's ZRLE implementation even kills tigervnc, they duplicate
# frames under certain conditions. Raw works ok
@encs = grep { $_->{name} ne 'ZRLE' } @encs;
}
if (!$self->jpeg) {
# according to the spec, RAW encoding is least preferred, so don't let
# loosy Tight JPEG be used over RAW unless explicitly requested
@encs = grep { $_->{name} ne 'Tight' and $_->{name} ne 'JPEG quality' } @encs;
}
$socket->print(
pack(
'CCn',
Expand Down Expand Up @@ -840,6 +852,9 @@ sub _receive_update ($self) {
$socket->read(my $data, $w * $h * $self->_bpp / 8) || die 'unexpected end of data';
$image->map_raw_data($data, $x, $y, $w, $h, $self->vncinfo);
}
elsif ($encoding_type == 7) { # Tight
$self->_receive_tight_encoding($x, $y, $w, $h);
}
elsif ($encoding_type == 16) { # ZRLE
$self->_receive_zrle_encoding($x, $y, $w, $h);
}
Expand Down Expand Up @@ -921,6 +936,63 @@ sub _receive_zrle_encoding ($self, $x, $y, $w, $h) {
return $res;
}

# wrapper to make testing easier
sub _read_socket ($socket, $data, $data_len, $offset) { return read($socket, $data, $data_len, $offset); } # uncoverable statement

sub _receive_tight_encoding ($self, $x, $y, $w, $h) {
my $socket = $self->socket;
my $image = $self->_framebuffer;

$socket->read(my $data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
my ($compression_control) = unpack('C', $data);
# FillCompression
if (($compression_control & 0xF0) == 0x80) {
my $data;
# special case for TPIXEL, otherwise identical to PIXEL
if ($self->_true_colour and $self->_bpp == 32 and $self->depth == 24) {
$socket->read($data, 3)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
$data = pack('CCCx', unpack('CCC', $data));
} else {
$socket->read($data, $self->_bpp / 8)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for compression control');
}
$image->fill_pixel($data, $self->vncinfo, $x, $y, $w, $h);
return;
}
# Only Fill (above) and JPEG for now; "Basic" unsupported
die "Unsupported compression $compression_control" if ($compression_control & 0xF0) != 0x90;

$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len) = unpack('C', $data);
if ($data_len & 0x80) {
$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len2) = unpack('C', $data);
$data_len = ($data_len & 0x7f) | ($data_len2 & 0x7f) << 7;
if ($data_len2 & 0x80) {
$socket->read($data, 1)
or OpenQA::Exception::VNCProtocolError->throw(error => 'short read for data len');
my ($data_len2) = unpack('C', $data);
$data_len |= $data_len2 << 14;
}
}

my $read_len = 0;
while ($read_len < $data_len) {
my $len = _read_socket($socket, $data, $data_len - $read_len, $read_len);
OpenQA::Exception::VNCProtocolError->throw(error => "short read for jpeg data $read_len - $data_len") unless $len;
$read_len += $len;
}
my $rect = tinycv::from_ppm($data);
OpenQA::Exception::VNCProtocolError->throw(error => "Invalid width/height of the rectangle (${w}x${h} != " . $rect->xres . "x" . $rect->yres . ")")
unless $w == $rect->xres and $h == $rect->yres;
$image->blend($rect, $x, $y);
$self->_framebuffer($image);
}

sub _receive_ikvm_encoding ($self, $encoding_type, $x, $y, $w, $h) {
my $socket = $self->socket;
my $image = $self->_framebuffer;
Expand Down
1 change: 1 addition & 0 deletions doc/backend_vars.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ GENERAL_HW_VNC_IP;string;;Hostname of the gadget's network. If not set, SSH cons
GENERAL_HW_VNC_PASSWORD;string;;Password for VNC server
GENERAL_HW_VNC_PORT;integer;5900;VNC Port number
GENERAL_HW_VNC_DEPTH;integer;16;Color depth for VNC server
GENERAL_HW_VNC_JPEG;integer;0;Advertise support for Tight JPEG encoding
GENERAL_HW_NO_SERIAL;boolean;;Don't use serial
GENERAL_HW_VIDEO_STREAM_URL;string;;Video stream URL (in ffmpeg's syntax) to receive, for example 'udp://@:5004' or '/dev/video0'. Using 'ustreamer:///dev/videoN' will use ustreamer from PiKVM instead of ffmpeg to read '/dev/videoN'. Ustreamer support requires pack("D") working, which rules out openSUSE 15.5's perl.
VIDEO_STREAM_PIPE_BUFFER_SIZE;integer;1680*1050*3+20;Buffer containing at least a single PPM frame for video capturing
Expand Down
70 changes: 70 additions & 0 deletions t/27-consoles-vnc.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ use Test::Most;
use Mojo::Base -strict, -signatures;
use utf8;

use Mojo::File qw(path);
use Test::Warnings qw(:all :report_warnings);
use Test::Exception;
use Test::Output qw(combined_like);
use Test::MockModule;
use Test::MockObject;
use Test::Mock::Time;
use FindBin '$Bin';
use File::Basename;
use lib "$Bin/../external/os-autoinst-common/lib";
use OpenQA::Test::TimeLimit '5';
use consoles::VNC;
Expand All @@ -33,6 +35,7 @@ my $s = Test::MockObject->new->set_true(qw(sockopt fileno print connected close
sub _setup_rfb_magic () { $s->set_series('mocked_read', 'RFB 003.006', pack('N', 1)) }
_setup_rfb_magic;
$s->mock(read => sub { $_[1] = $s->mocked_read; length $_[1] });
$vnc_mock->redefine(_read_socket => sub { substr($_[1], $_[3], $_[2]) = $s->mocked_read; length $_[1] });
$s->mock($_ => sub { push @printed, $_[1] }) for qw(print write);
$inet_mock->redefine(new => $s);
$vnc_mock->noop('_server_initialization');
Expand Down Expand Up @@ -243,6 +246,54 @@ subtest 'update framebuffer' => sub {
$s->set_series(mocked_read => $update_message, $one_rectangle, $ikvm_encoding, $ikvm_specific_data, $actual_image_data);
$c->update_framebuffer;
is scalar @printed, 1, 'no further image requested' or diag explain \@printed;

my $of_type_tight_with_coordinates_12_42_4_4 = pack(nnnnNC => 12, 42, 4, 4, 7);
my $fill_compression = pack("C", 0x80);
subtest 'Tight encoding, FillCompression' => sub {
$c->_do_endian_conversion($machine_is_big_endian); # assume server is little-endian
$vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16);
$gray_pixel = pack(CCCC => 31, 37, 41, 0); # dark prime grey
$s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_12_42_4_4, $fill_compression, $gray_pixel);
$c->_framebuffer(undef)->width(1024)->height(512)->vncinfo($vncinfo);
ok $c->update_framebuffer, 'truthy return value for successful pixel update';
($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44);
is $blue, 41, 'pixel data updated in framebuffer (blue)';
is $green, 37, 'pixel data updated in framebuffer (green)';
is $red, 31, 'pixel data updated in framebuffer (red)';
};

subtest 'Tight encoding, FillCompression, TPIXEL' => sub {
$c->_do_endian_conversion($machine_is_big_endian); # assume server is little-endian
$c->depth(24);
$c->_true_colour(1);
$vncinfo = tinycv::new_vncinfo($c->_do_endian_conversion, $c->_true_colour, $c->_bpp / 8, 255, 0, 255, 8, 255, 16);
$gray_pixel = pack(CCCC => 31, 37, 41); # dark prime grey
$s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_12_42_4_4, $fill_compression, $gray_pixel);
$c->_framebuffer(undef)->width(1024)->height(512)->vncinfo($vncinfo);
ok $c->update_framebuffer, 'truthy return value for successful pixel update';
($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44);
is $blue, 41, 'pixel data updated in framebuffer (blue)';
is $green, 37, 'pixel data updated in framebuffer (green)';
is $red, 31, 'pixel data updated in framebuffer (red)';
};

subtest 'Tight encoding, JpegCompression' => sub {
my $jpeg_data = path(dirname(__FILE__) . '/data/frame1.jpeg')->slurp;
my $of_type_tight_with_coordinates_0_0_1024_768 = pack(nnnnNC => 0, 0, 1024, 768, 7);
my $jpeg_compression = pack("C", 0x90);
my $data_len = length($jpeg_data);
my $data_len1 = pack("C", ($data_len & 0x7f) | 0x80);
my $data_len2 = pack("C", (($data_len >> 7) & 0x7f) | 0x80);
my $data_len3 = pack("C", ($data_len >> 14));

$s->set_series(mocked_read => $update_message, $one_rectangle, $of_type_tight_with_coordinates_0_0_1024_768, $jpeg_compression, $data_len1, $data_len2, $data_len3, $jpeg_data);
$c->_framebuffer(undef)->width(1024)->height(768);
ok $c->update_framebuffer, 'truthy return value for successful pixel update';
($blue, $green, $red) = $c->_framebuffer->get_pixel(14, 44);
is $blue, 0x3e, 'pixel data updated in framebuffer (blue)';
is $green, 0x84, 'pixel data updated in framebuffer (green)';
is $red, 0x01, 'pixel data updated in framebuffer (red)';
};
};

subtest 'read special messages/encodings' => sub {
Expand Down Expand Up @@ -368,6 +419,25 @@ subtest 'server initialization' => sub {
pack(N => -261), # VNC_ENCODING_LED_STATE
);
is_deeply \@printed, \@expected, 'pixel format and encodings replied' or diag explain \@printed;

# test with jpeg enabled
@printed = ();
$s->set_series(mocked_read => $server_init);
$c->jpeg(1);
$c->_server_initialization;
# expect params for 16-bit depth being replied as setpixelformat
@expected = (
pack(CCCCCCCCnnnCCCCCC => 0, 0, 0, 0, @params, 0, 0, 0), # setpixelformat
pack(CCn => 2, 0, 7), # seven supported encodings (no ZRLE due to dell flag)
pack(N => 0000), # raw
pack(N => 0007), # Tight
pack(N => -24), # JPEG quality
pack(N => -223), # DesktopSize
pack(N => -224), # VNC_ENCODING_LAST_RECT
pack(N => -257), # VNC_ENCODING_POINTER_TYPE_CHANGE
pack(N => -261), # VNC_ENCODING_LED_STATE
);
is_deeply \@printed, \@expected, 'pixel format and encodings replied' or diag explain \@printed;
};

subtest 'login on real VNC server via vnctest, request and receive frame buffer' => sub {
Expand Down
Binary file added t/data/frame1.jpeg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 0195131

Please sign in to comment.