vo_opengl: initial smoothmotion support

This uses a very simple algorithm that blends the resulting frames. The code
will need changes since the blending must be done in linear light to improve
image quality and color correctness.

video/out/gl_video.c

@@ -121,6 +121,13 @@ struct fbotex {
     int vp_x, vp_y, vp_w, vp_h; // viewport of fbo / used part of the texture
 };
 
+struct fbosurface {
+    struct fbotex fbotex;
+    int64_t pts;
+};
+
+#define FBOSURFACES_MAX 2
+
 struct gl_video {
     GL *gl;
 
@@ -136,7 +143,7 @@ struct gl_video {
     GLuint vao;
 
     GLuint osd_programs[SUBBITMAP_COUNT];
-    GLuint indirect_program, scale_sep_program, final_program;
+    GLuint indirect_program, scale_sep_program, final_program, inter_program;
 
     struct osd_state *osd_state;
     struct mpgl_osd *osd;
@@ -175,6 +182,8 @@ struct gl_video {
 
     struct fbotex indirect_fbo;         // RGB target
     struct fbotex scale_sep_fbo;        // first pass when doing 2 pass scaling
+    struct fbosurface surfaces[FBOSURFACES_MAX];
+    size_t surface_num;
 
     // state for luma (0) and chroma (1) scalers
     struct scaler scalers[2];
@@ -537,6 +546,38 @@ static void fbotex_uninit(struct gl_video *p, struct fbotex *fbo)
     }
 }
 
+static void fbosurfaces_uninit(struct gl_video *p, struct fbosurface *surfaces)
+{
+
+    for (int i = 0; i < FBOSURFACES_MAX; i++)
+        if (surfaces[i].fbotex.fbo)
+            fbotex_uninit(p, &surfaces[i].fbotex);
+}
+
+static void fbosurfaces_init(struct gl_video *p, struct fbosurface *surfaces,
+                          int w, int h, GLenum iformat)
+{
+    fbosurfaces_uninit(p, surfaces);
+    for (int i = 0; i < FBOSURFACES_MAX; i++)
+        if (!surfaces[i].fbotex.fbo && w > 0 && h > 0)
+            fbotex_init(p, &surfaces[i].fbotex, w, h, iformat);
+}
+
+
+static void fbosurface_bind(struct gl_video *p, GLuint *texs, int i)
+{
+    GL *gl = p->gl;
+    struct fbotex *fbotex = &p->surfaces[p->surface_num].fbotex;
+    gl->ActiveTexture(GL_TEXTURE0 + i);
+    gl->BindTexture(p->gl_target, p->surfaces[p->surface_num].fbotex.texture);
+    texs[i] = fbotex->texture;
+}
+
+static size_t fbosurface_next(struct gl_video *p)
+{
+    return (p->surface_num + 1) % FBOSURFACES_MAX;
+}
+
 static void matrix_ortho2d(float m[3][3], float x0, float x1,
                            float y0, float y1)
 {
@@ -706,6 +747,7 @@ static void update_all_uniforms(struct gl_video *p)
     update_uniforms(p, p->indirect_program);
     update_uniforms(p, p->scale_sep_program);
     update_uniforms(p, p->final_program);
+    update_uniforms(p, p->inter_program);
 }
 
 #define SECTION_HEADER "#!section "
@@ -997,6 +1039,7 @@ static void compile_shaders(struct gl_video *p)
 
     char *header_conv = talloc_strdup(tmp, "");
     char *header_final = talloc_strdup(tmp, "");
+    char *header_inter = talloc_strdup(tmp, "");
     char *header_sep = NULL;
 
     if (p->image_desc.id == IMGFMT_NV12 || p->image_desc.id == IMGFMT_NV21) {
@@ -1046,6 +1089,8 @@ static void compile_shaders(struct gl_video *p)
         shader_setup_scaler(&header_final, &p->scalers[0], -1);
     }
 
+    shader_def_opt(&header_inter, "USE_LINEAR_INTERPOLATION", 1);
+
     // We want to do scaling in linear light. Scaling is closely connected to
     // texture sampling due to how the shader is structured (or if GL bilinear
     // scaling is used). The purpose of the "indirect" pass is to convert the
@@ -1103,6 +1148,10 @@ static void compile_shaders(struct gl_video *p)
     p->final_program =
         create_program(p, "final", header_final, vertex_shader, s_video);
 
+    header_inter = t_concat(tmp, header, header_inter);
+    p->inter_program =
+        create_program(p, "inter", header_inter, vertex_shader, s_video);
+
     debug_check_gl(p, "shader compilation");
 
     talloc_free(tmp);
@@ -1123,6 +1172,7 @@ static void delete_shaders(struct gl_video *p)
     delete_program(gl, &p->indirect_program);
     delete_program(gl, &p->scale_sep_program);
     delete_program(gl, &p->final_program);
+    delete_program(gl, &p->inter_program);
 }
 
 static void get_scale_factors(struct gl_video *p, double xy[2])
@@ -1544,6 +1594,7 @@ static void uninit_video(struct gl_video *p)
 
     fbotex_uninit(p, &p->indirect_fbo);
     fbotex_uninit(p, &p->scale_sep_fbo);
+    fbosurfaces_uninit(p, p->surfaces);
 }
 
 static void change_dither_trafo(struct gl_video *p)
@@ -1573,6 +1624,8 @@ struct pass {
     // If true, render source (f) to dst, instead of the full dest. fbo viewport
     bool use_dst;
     struct mp_rect dst;
+    bool use_src;
+    struct mp_rect src;
     int flags; // for write_quad
     bool render_stereo;
 };
@@ -1603,6 +1656,9 @@ static void handle_pass(struct gl_video *p, struct pass *chain,
         .y1 = chain->f.vp_y + chain->f.vp_h,
     };
 
+    if (chain->use_src)
+        src = chain->src;
+
     struct mp_rect dst = {-1, -1, 1, 1};
     if (chain->use_dst)
         dst = chain->dst;
@@ -1651,7 +1707,7 @@ static void handle_pass(struct gl_video *p, struct pass *chain,
 }
 
 // (fbo==0 makes BindFramebuffer select the screen backbuffer)
-void gl_video_render_frame(struct gl_video *p, int fbo)
+void gl_video_render_frame(struct gl_video *p, int fbo, struct frame_timing *t)
 {
     GL *gl = p->gl;
     struct video_image *vimg = &p->image;
@@ -1721,7 +1777,67 @@ void gl_video_render_frame(struct gl_video *p, int fbo)
                 | (vimg->image_flipped ? 4 : 0);
     chain.render_stereo = true;
 
-    handle_pass(p, &chain, &screen, p->final_program);
+    if (!t) {
+        handle_pass(p, &chain, &screen, p->final_program);
+    } else {
+        GLuint imgtexsurfaces[4] = {0};
+        double inter_coeff = 0.0;
+        int64_t prev_pts = p->surfaces[fbosurface_next(p)].pts;
+
+        if (prev_pts != t->pts) {
+            MP_STATS(p, "new-pts");
+            struct fbotex *fbotex = &p->surfaces[p->surface_num].fbotex;
+            handle_pass(p, &chain, fbotex, p->final_program);
+            p->surfaces[p->surface_num].pts = t->pts;
+            fbosurface_bind(p, imgtexsurfaces, 0);
+            p->surface_num = fbosurface_next(p);
+            fbosurface_bind(p, imgtexsurfaces, 1);
+            gl->ActiveTexture(GL_TEXTURE0);
+            MP_DBG(p, "frame ppts: %lld, pts: %lld, vsync: %lld, DIFF: %lld\n",
+                      (long long)prev_pts, (long long)t->pts,
+                      (long long)t->next_vsync, (long long)t->next_vsync - t->pts);
+            if (prev_pts < t->next_vsync && t->pts > t->next_vsync) {
+                double N = t->next_vsync - prev_pts;
+                double P = t->pts - prev_pts;
+                double prev_pts_component = N / P;
+                inter_coeff = 1 - prev_pts_component;
+                MP_DBG(p, "inter frame ppts: %lld, pts: %lld, "
+                       "vsync: %lld, mix: %f\n",
+                       (long long)prev_pts, (long long)t->pts,
+                       (long long)t->next_vsync, inter_coeff);
+                MP_STATS(p, "frame-mix");
+            }
+        } else {
+            MP_STATS(p, "old-pts");
+            p->surface_num = fbosurface_next(p);
+            fbosurface_bind(p, imgtexsurfaces, 0);
+            p->surface_num = fbosurface_next(p);
+            fbosurface_bind(p, imgtexsurfaces, 1);
+            gl->ActiveTexture(GL_TEXTURE0);
+        }
+
+        // the value is scaled to fit in the graph with the completely
+        // unrelated "phase" value (which is stupid)
+        MP_STATS(p, "value-timed %lld %f mix-value",
+                 (long long)t->pts, inter_coeff * 10000);
+
+        chain.use_dst = true;
+        chain.dst = p->dst_rect;
+        chain.use_src = true;
+        chain.src = p->dst_rect;
+        chain.flags = (1 << 2); // (needs flip, wut)
+        chain.f.texture = imgtexsurfaces[0];
+        chain.f.tex_w = p->surfaces[p->surface_num].fbotex.tex_w;
+        chain.f.tex_h = p->surfaces[p->surface_num].fbotex.tex_h;
+
+        gl->UseProgram(p->inter_program);
+        GLint loc = gl->GetUniformLocation(p->inter_program, "inter_coeff");
+        if (loc >= 0) {
+            gl->Uniform1f(loc, inter_coeff);
+        }
+
+        handle_pass(p, &chain, &screen, p->inter_program);
+    }
 
     gl->UseProgram(0);
 
@@ -1739,9 +1855,10 @@ void gl_video_render_frame(struct gl_video *p, int fbo)
 
 static void update_window_sized_objects(struct gl_video *p)
 {
+    int w = p->dst_rect.x1 - p->dst_rect.x0;
+    int h = p->dst_rect.y1 - p->dst_rect.y0;
+
     if (p->scale_sep_program) {
-        int w = p->dst_rect.x1 - p->dst_rect.x0;
-        int h = p->dst_rect.y1 - p->dst_rect.y0;
         if ((p->image_params.rotate % 180) == 90)
             MPSWAP(int, w, h);
         if (h > p->scale_sep_fbo.tex_h) {
@@ -1755,6 +1872,10 @@ static void update_window_sized_objects(struct gl_video *p)
         p->scale_sep_fbo.vp_w = p->image_w;
         p->scale_sep_fbo.vp_h = h;
     }
+
+    if (p->opts.smoothmotion) {
+        fbosurfaces_init(p, p->surfaces, p->vp_w, p->vp_h, p->opts.fbo_format);
+    }
 }
 
 static void check_resize(struct gl_video *p)
@@ -2511,7 +2632,7 @@ void gl_video_resize_redraw(struct gl_video *p, int w, int h)
 {
     p->vp_w = w;
     p->vp_h = h;
-    gl_video_render_frame(p, 0);
+    gl_video_render_frame(p, 0, NULL);
 }
 
 void gl_video_set_hwdec(struct gl_video *p, struct gl_hwdec *hwdec)

video/out/gl_video.h

@@ -68,7 +68,7 @@ void gl_video_config(struct gl_video *p, struct mp_image_params *params);
 void gl_video_set_output_depth(struct gl_video *p, int r, int g, int b);
 void gl_video_set_lut3d(struct gl_video *p, struct lut3d *lut3d);
 void gl_video_upload_image(struct gl_video *p, struct mp_image *img);
-void gl_video_render_frame(struct gl_video *p, int fbo);
+void gl_video_render_frame(struct gl_video *p, int fbo, struct frame_timing *t);
 struct mp_image *gl_video_download_image(struct gl_video *p);
 void gl_video_resize(struct gl_video *p, struct mp_rect *window,
                      struct mp_rect *src, struct mp_rect *dst,

video/out/gl_video_shaders.glsl

@@ -169,6 +169,7 @@ uniform float dither_center;
 uniform float filter_param1_l;
 uniform float filter_param1_c;
 uniform vec2 dither_size;
+uniform float inter_coeff;
 
 in vec2 texcoord;
 DECLARE_FRAGPARMS
@@ -325,7 +326,15 @@ void main() {
 #ifndef USE_CONV
 #define USE_CONV 0
 #endif
-#if USE_CONV == CONV_PLANAR
+#ifndef USE_LINEAR_INTERPOLATION
+#define USE_LINEAR_INTERPOLATION 0
+#endif
+#if USE_LINEAR_INTERPOLATION == 1
+    vec4 acolor = mix(texture(texture0, chr_texcoord),
+                      // vec4(1.0, 0.0, 0.0, 1.0),
+                      texture(texture1, chr_texcoord),
+                      inter_coeff);
+#elif USE_CONV == CONV_PLANAR
     vec4 acolor = vec4(SAMPLE_L(texture0, textures_size[0], texcoord).r,
                        SAMPLE_C(texture1, textures_size[1], chr_texcoord).r,
                        SAMPLE_C(texture2, textures_size[2], chr_texcoord).r,

video/out/vo.c

@@ -569,7 +569,15 @@ static bool render_frame(struct vo *vo)
     int64_t next_vsync = prev_vsync + in->vsync_interval;
     int64_t end_time = pts + duration;
 
-    MP_STATS(vo, "next_vsync %lld, pts %lld, img %d\n", next_vsync, pts, !!img);
+    MP_DBG(vo, "next_vsync %lld, pts %lld, img %d\n", (long long)next_vsync,
+           (long long)pts, !!img);
+    MP_STATS(vo, "event-timed %lld prev_vsync", (long long)prev_vsync);
+    MP_STATS(vo, "event-timed %lld next_vsync", (long long)next_vsync);
+    if (img) {
+        MP_STATS(vo, "event-timed %lld pts", (long long)pts);
+    } else {
+        MP_STATS(vo, "event-timed %lld pts-no-img", (long long)pts);
+    }
 
     if (!(vo->global->opts->frame_dropping & 1) || !in->hasframe_rendered ||
         vo->driver->untimed || vo->driver->encode)
@@ -628,6 +636,7 @@ static bool render_frame(struct vo *vo)
         MP_DBG(vo, "phase: %ld\n", phase);
         MP_STATS(vo, "value %ld phase", phase);
 
+        MP_STATS(vo, "display");
         MP_STATS(vo, "end video");
 
         pthread_mutex_lock(&in->lock);
@@ -692,6 +701,7 @@ static void *vo_thread(void *ptr)
     mpthread_set_name("vo");
 
     int r = vo->driver->preinit(vo) ? -1 : 0;
+    vo->driver->control(vo, VOCTRL_GET_VSYNC_TIMED, &in->vsync_timed);
     mp_rendezvous(vo, r); // init barrier
     if (r < 0)
         return NULL;

video/out/vo.h

@@ -108,6 +108,8 @@ enum mp_voctrl {
     VOCTRL_GET_PREF_DEINT,              // int*
 
     VOCTRL_SET_LIBMPV_OPENGL_CB_CONTEXT,// struct mpv_opengl_cb_context*
+
+    VOCTRL_GET_VSYNC_TIMED,             // bool*
 };
 
 // VOCTRL_SET_EQUALIZER

video/out/vo_opengl.c

@@ -71,6 +71,7 @@ struct gl_priv {
     int use_gl_debug;
     int allow_sw;
     int swap_interval;
+    int smoothmotion;
     char *backend;
 
     int vo_flipped;
@@ -165,7 +166,7 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     mpgl_lock(p->glctx);
 
     gl_video_upload_image(p->renderer, mpi);
-    gl_video_render_frame(p->renderer, 0);
+    gl_video_render_frame(p->renderer, 0, NULL);
 
     // The playloop calls this last before waiting some time until it decides
     // to call flip_page(). Tell OpenGL to start execution of the GPU commands
@@ -190,11 +191,13 @@ static void draw_image_timed(struct vo *vo, mp_image_t *mpi,
     mpgl_lock(p->glctx);
 
     if (mpi) gl_video_upload_image(p->renderer, mpi);
-    gl_video_render_frame(p->renderer);
+    gl_video_render_frame(p->renderer, 0, t);
 
-    MP_STATS(vo, "prev_vsync: %lld; next_vsync %lld, pts %lld, img %d, "
-             "1: %lld 2: %lld\n", t->prev_vsync, t->next_vsync, t->pts, !!mpi,
-             t->pts - t->prev_vsync, t->pts - t->next_vsync);
+    MP_DBG(vo, "prev_vsync: %lld; next_vsync %lld, pts %lld, img %d, "
+           "1: %lld 2: %lld\n", (long long)t->prev_vsync,
+           (long long)t->next_vsync, (long long)t->pts, !!mpi,
+           (long long)(t->pts - t->prev_vsync),
+           (long long)(t->pts - t->next_vsync));
 
     // The playloop calls this last before waiting some time until it decides
     // to call flip_page(). Tell OpenGL to start execution of the GPU commands
@@ -391,13 +394,16 @@ static int control(struct vo *vo, uint32_t request, void *data)
         return true;
     case VOCTRL_REDRAW_FRAME:
         mpgl_lock(p->glctx);
-        gl_video_render_frame(p->renderer, 0);
+        gl_video_render_frame(p->renderer, 0, NULL);
         mpgl_unlock(p->glctx);
         return true;
     case VOCTRL_SET_COMMAND_LINE: {
         char *arg = data;
         return reparse_cmdline(p, arg);
     }
+    case VOCTRL_GET_VSYNC_TIMED:
+        *(bool *)data = p->smoothmotion;
+        return VO_TRUE;
     }
 
     mpgl_lock(p->glctx);
@@ -479,6 +485,7 @@ static int preinit(struct vo *vo)
 static const struct m_option options[] = {
     OPT_FLAG("glfinish", use_glFinish, 0),
     OPT_FLAG("waitvsync", waitvsync, 0),
+    OPT_FLAG("smoothmotion", smoothmotion, 0),
     OPT_INT("swapinterval", swap_interval, 0, OPTDEF_INT(1)),
     OPT_FLAG("debug", use_gl_debug, 0),
     OPT_STRING_VALIDATE("backend", backend, 0, mpgl_validate_backend_opt),

video/out/vo_opengl_cb.c

@@ -217,7 +217,7 @@ int mpv_opengl_cb_render(struct mpv_opengl_cb_context *ctx, int fbo, int vp[4])
     if (mpi)
         gl_video_upload_image(ctx->renderer, mpi);
 
-    gl_video_render_frame(ctx->renderer, fbo);
+    gl_video_render_frame(ctx->renderer, fbo, NULL);
 
     gl_video_unset_gl_state(ctx->renderer);