Minimal VNC server implementation

app/meson.build

@@ -83,6 +83,11 @@ if v4l2_support
     src += [ 'src/v4l2_sink.c' ]
 endif
 
+vnc_support = get_option('vnc') and host_machine.system() == 'linux'
+if vnc_support
+    src += [ 'src/vnc_sink.c' ]
+endif
+
 usb_support = get_option('usb')
 if usb_support
     src += [
@@ -113,6 +118,10 @@ if not crossbuild_windows
     if v4l2_support
         dependencies += dependency('libavdevice')
     endif
+    if vnc_support
+        dependencies += dependency('libswscale')
+        dependencies += dependency('libvncserver')
+    endif
 
     if usb_support
         dependencies += dependency('libusb-1.0')
@@ -214,6 +223,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '
 # enable V4L2 support (linux only)
 conf.set('HAVE_V4L2', v4l2_support)
 
+# enable libvnc support
+conf.set('HAVE_VNC', vnc_support)
+
 # enable HID over AOA support (linux only)
 conf.set('HAVE_USB', usb_support)
 

app/src/cli.c

@@ -72,6 +72,7 @@ enum {
     OPT_REQUIRE_AUDIO,
     OPT_AUDIO_BUFFER,
     OPT_AUDIO_OUTPUT_BUFFER,
+    OPT_VNC_SERVER,
 };
 
 struct sc_option {
@@ -669,6 +670,11 @@ static const struct sc_option options[] = {
         .text = "Set the initial window height.\n"
                 "Default is 0 (automatic).",
     },
+    {
+        .longopt_id = OPT_VNC_SERVER,
+        .longopt = "vnc-server",
+        .text = "Enable VNC server.",
+    },
 };
 
 static const struct sc_shortcut shortcuts[] = {
@@ -1861,6 +1867,14 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case OPT_VNC_SERVER:
+#ifdef HAVE_VNC
+                opts->vnc_server = true;
+#else
+                LOGE("VNC (--vnc-server) is disabled.");
+                return false;
+#endif
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;
@@ -1889,10 +1903,22 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         return false;
     }
 
+    bool has_sink = opts->record_filename;
 #ifdef HAVE_V4L2
-    if (!opts->display && !opts->record_filename && !opts->v4l2_device) {
-        LOGE("-N/--no-display requires either screen recording (-r/--record)"
-             " or sink to v4l2loopback device (--v4l2-sink)");
+    has_sink |= (opts->v4l2_device != NULL);
+#endif
+#ifdef HAVE_VNC
+    has_sink |= opts->vnc_server;
+#endif
+    if (!opts->display && !has_sink) {
+        LOGE("-N/--no-display requires at least one of the following to be set: ");
+        LOGE("* screen recording (-r/--record)");
+#ifdef HAVE_V4L2
+        LOGE("* sink to v4l2loopback device (--v4l2-sink)");
+#endif
+#ifdef HAVE_VNC
+        LOGE("* setting up a VNC server (--vnc-server)");
+#endif
         return false;
     }
 
@@ -1914,12 +1940,6 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         LOGE("V4L2 buffer value without V4L2 sink\n");
         return false;
     }
-#else
-    if (!opts->display && !opts->record_filename) {
-        LOGE("-N/--no-display requires screen recording (-r/--record)");
-        return false;
-    }
-#endif
 
     if (opts->audio && !opts->display && !opts->record_filename) {
         LOGI("No display and no recording: audio disabled");

app/src/options.c

@@ -53,6 +53,7 @@ const struct scrcpy_options scrcpy_options_default = {
     .always_on_top = false,
     .control = true,
     .display = true,
+    .vnc_server = false,
     .turn_screen_off = false,
     .key_inject_mode = SC_KEY_INJECT_MODE_MIXED,
     .window_borderless = false,

app/src/options.h

@@ -136,6 +136,7 @@ struct scrcpy_options {
     bool always_on_top;
     bool control;
     bool display;
+    bool vnc_server;
     bool turn_screen_off;
     enum sc_key_inject_mode key_inject_mode;
     bool window_borderless;

app/src/scrcpy.c

@@ -38,6 +38,9 @@
 #ifdef HAVE_V4L2
 # include "v4l2_sink.h"
 #endif
+#ifdef HAVE_VNC
+#include "vnc_sink.h"
+#endif
 
 struct scrcpy {
     struct sc_server server;
@@ -49,6 +52,9 @@ struct scrcpy {
     struct sc_decoder audio_decoder;
     struct sc_recorder recorder;
     struct sc_delay_buffer display_buffer;
+#ifdef HAVE_VNC
+    struct sc_vnc_sink vnc_sink;
+#endif
 #ifdef HAVE_V4L2
     struct sc_v4l2_sink v4l2_sink;
     struct sc_delay_buffer v4l2_buffer;
@@ -310,6 +316,9 @@ scrcpy(struct scrcpy_options *options) {
     bool recorder_started = false;
 #ifdef HAVE_V4L2
     bool v4l2_sink_initialized = false;
+#endif
+#ifdef HAVE_VNC
+    bool vnc_sink_initialized = false;
 #endif
     bool video_demuxer_started = false;
     bool audio_demuxer_started = false;
@@ -451,7 +460,7 @@ scrcpy(struct scrcpy_options *options) {
                         &audio_demuxer_cbs, options);
     }
 
-    bool needs_video_decoder = options->display;
+    bool needs_video_decoder = options->display || options->vnc_server;
     bool needs_audio_decoder = options->audio && options->display;
 #ifdef HAVE_V4L2
     needs_video_decoder |= !!options->v4l2_device;
@@ -694,6 +703,17 @@ scrcpy(struct scrcpy_options *options) {
                                      &s->audio_player.frame_sink);
         }
     }
+#ifdef HAVE_VNC
+    if (options->vnc_server) {
+        if (!sc_vnc_sink_init(&s->vnc_sink, "my vnc server", controller)) {
+            printf("bad vnc init \n");
+            goto end;
+        }
+        vnc_sink_initialized = true;
+        struct sc_frame_source *src = &s->video_decoder.frame_source;
+        sc_frame_source_add_sink(src, &s->vnc_sink.frame_sink);
+    }
+#endif
 
 #ifdef HAVE_V4L2
     if (options->v4l2_device) {
@@ -786,6 +806,11 @@ scrcpy(struct scrcpy_options *options) {
         sc_v4l2_sink_destroy(&s->v4l2_sink);
     }
 #endif
+#ifdef HAVE_VNC
+    if (vnc_sink_initialized) {
+        sc_vnc_sink_destroy(&s->vnc_sink);
+    }
+#endif
 
 #ifdef HAVE_USB
     if (aoa_hid_initialized) {

app/src/vnc_sink.c

@@ -0,0 +1,132 @@
+#include "vnc_sink.h"
+
+#include <string.h>
+
+#include "util/log.h"
+#include "util/str.h"
+
+/** Downcast frame_sink to sc_vnc_sink */
+#define DOWNCAST(SINK) container_of(SINK, struct sc_vnc_sink, frame_sink)
+
+
+static bool
+sc_vnc_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) {
+    assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P);
+    (void) sink;
+    (void) ctx;
+    return true;
+}
+
+static void
+sc_vnc_frame_sink_close(struct sc_frame_sink *sink) {
+    (void) sink;
+}
+
+static bool
+sc_vnc_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
+    struct sc_vnc_sink *vnc = DOWNCAST(sink);
+    // XXX: ideally this would get "damage" regions from the decoder
+    // to prevent marking the entire screen as modified if only a small
+    // part changed
+    if(frame->width != vnc->scrWidth || frame->height != vnc->scrHeight) {
+        if(vnc->ctx) {
+            sws_freeContext(vnc->ctx);
+            vnc->ctx = NULL;
+        }
+    }
+    if(vnc->ctx == NULL) {
+        vnc->scrWidth = frame->width;
+        vnc->scrHeight = frame->height;
+        vnc->ctx = sws_getContext(frame->width, frame->height, AV_PIX_FMT_YUV420P,
+                                  frame->width, frame->height, AV_PIX_FMT_RGBA,
+                                  0, 0, 0, 0);
+        if(vnc->ctx == NULL) {
+            LOGE("could not make context");
+            return false;
+        }
+        char *currentFrameBuffer = vnc->screen->frameBuffer;
+        char *newFrameBuffer = (char *)malloc(vnc->scrWidth*vnc->scrHeight*vnc->bpp);
+        rfbNewFramebuffer(vnc->screen, newFrameBuffer, vnc->scrWidth, vnc->scrHeight, 8, 3, vnc->bpp);
+        free(currentFrameBuffer);
+    }
+    assert(vnc->ctx != NULL);
+
+    int linesize[1] = {frame->width*vnc->bpp};
+    uint8_t *const data[1] = {(uint8_t*)vnc->screen->frameBuffer};
+    sws_scale(vnc->ctx, (const uint8_t * const *)frame->data, frame->linesize, 0, frame->height, data, linesize);
+
+    rfbMarkRectAsModified(vnc->screen, 0, 0, frame->width, frame->height);
+    return true;
+}
+
+bool
+sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_controller *controller) {
+    uint8_t placeholder_width = 32;
+    uint8_t placeholder_height = 32;
+    static const struct sc_frame_sink_ops ops = {
+        .open = sc_vnc_frame_sink_open,
+        .close = sc_vnc_frame_sink_close,
+        .push = sc_vnc_frame_sink_push,
+    };
+
+    vs->frame_sink.ops = &ops;
+    vs->bpp = 4;
+    vs->screen = rfbGetScreen(0, NULL, placeholder_width, placeholder_height, 8, 3, vs->bpp);
+    vs->screen->desktopName = device_name;
+    vs->screen->alwaysShared = true;
+    vs->screen->frameBuffer = (char *)malloc(placeholder_width * placeholder_height * vs->bpp);
+    vs->screen->ptrAddEvent = ptr_add_event;
+    vs->screen->screenData = vs;
+    vs->was_down = false;
+    vs->controller = controller;
+    rfbInitServer(vs->screen);
+    rfbRunEventLoop(vs->screen, -1, true); // TODO: integrate into proper lifecycle
+    return true;
+}
+
+void
+sc_vnc_sink_destroy(struct sc_vnc_sink *vs) {
+    if(vs->screen) {
+        rfbShutdownServer(vs->screen, true);
+        free(vs->screen->frameBuffer);
+        rfbScreenCleanup(vs->screen);
+    }
+    if(vs->ctx) {
+        sws_freeContext(vs->ctx);
+    }
+}
+
+void
+ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl) {
+    struct sc_vnc_sink *vnc = cl->screen->screenData;
+    // buttonMask is 3 bits: MOUSE_RIGHT | MOUSE_MIDDLE | MOUSE_LEFT
+    // value of 1 in that bit indicates it's being pressed; 0 indicates released
+
+    // TODO: only doing left click
+    bool up = (buttonMask & 0x1) == 0;
+    struct sc_control_msg msg;
+    struct sc_size screen_size = {vnc->scrWidth, vnc->scrHeight};
+    struct sc_point point = {x, y};
+
+    msg.type = SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT;
+    if(vnc->was_down && !up) {
+        msg.inject_touch_event.action = AMOTION_EVENT_ACTION_MOVE;
+    } else {
+        msg.inject_touch_event.action = up ? AMOTION_EVENT_ACTION_UP : AMOTION_EVENT_ACTION_DOWN;
+    }
+    msg.inject_touch_event.position.screen_size = screen_size;
+    msg.inject_touch_event.position.point = point;
+    msg.inject_touch_event.pointer_id = POINTER_ID_VIRTUAL_FINGER;
+    // TODO: how to decide vs POINTER_ID_VIRTUAL_MOUSE?
+
+    msg.inject_touch_event.pressure = up ? 0.0f : 1.0f;
+    msg.inject_touch_event.action_button = 0;
+    msg.inject_touch_event.buttons = 0;
+
+    if (!sc_controller_push_msg(vnc->controller, &msg)) {
+        LOGW("Could not request 'inject virtual finger event'");
+    }
+
+    rfbDefaultPtrAddEvent(buttonMask, x, y, cl);
+    vnc->was_down = !up;
+}

app/src/vnc_sink.h

@@ -0,0 +1,40 @@
+#ifndef SC_VNC_SINK_H
+#define SC_VNC_SINK_H
+
+#include "common.h"
+
+#include <libswscale/swscale.h>
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <rfb/rfb.h>
+
+#include "coords.h"
+#include "control_msg.h"
+#include "controller.h"
+#include "trait/frame_sink.h"
+#include "frame_buffer.h"
+#include "util/tick.h"
+
+struct sc_vnc_sink {
+    struct sc_frame_sink frame_sink; // frame sink trait
+    struct sc_controller *controller;
+
+    struct SwsContext * ctx;
+    rfbScreenInfoPtr screen;
+    uint16_t scrWidth;
+    uint16_t scrHeight;
+    uint8_t bpp;
+
+    bool was_down;
+    char *device_name;
+};
+
+bool
+sc_vnc_sink_init(struct sc_vnc_sink *vs, const char *device_name, struct sc_controller *controller);
+
+void
+sc_vnc_sink_destroy(struct sc_vnc_sink *vs);
+
+void
+ptr_add_event(int buttonMask, int x, int y, rfbClientPtr cl);
+#endif

meson_options.txt

@@ -5,4 +5,5 @@ option('portable', type: 'boolean', value: false, description: 'Use scrcpy-serve
 option('server_debugger', type: 'boolean', value: false, description: 'Run a server debugger and wait for a client to be attached')
 option('server_debugger_method', type: 'combo', choices: ['old', 'new'], value: 'new', description: 'Select the debugger method (Android < 9: "old", Android >= 9: "new")')
 option('v4l2', type: 'boolean', value: true, description: 'Enable V4L2 feature when supported')
+option('vnc', type: 'boolean', value: true, description: 'Enable VNC server feature when supported')
 option('usb', type: 'boolean', value: true, description: 'Enable HID/OTG features when supported')