avcodec/nvenc: only unregister input resources when absolutely needed

This reverts nvenc to old behaviour, which in some super rare edge cases
performs better.
The implication of this is that any potential API user who relies on
nvenc cleaning up every frames device resources after it's done using
them will now have to make a custom build of ffmpeg with the
compile-time define NVENC_UNREGISTER_IMMEDIATELY set to 1.

Pretty much nobody should need to set this, since the normal usage
pattern of nvenc usually makes sure all frames are allocated from a
common pool which reuses device resources as much as possible and holds
on to them until the encode session ends.

Author: BtbN

libavcodec/nvenc.c

@@ -1560,19 +1560,23 @@ static int nvenc_find_free_reg_resource(AVCodecContext *avctx)
     NV_ENCODE_API_FUNCTION_LIST *p_nvenc = &dl_fn->nvenc_funcs;
     NVENCSTATUS nv_status;
 
-    int i;
+    int i, first_round;
 
     if (ctx->nb_registered_frames == FF_ARRAY_ELEMS(ctx->registered_frames)) {
-        for (i = 0; i < ctx->nb_registered_frames; i++) {
-            if (!ctx->registered_frames[i].mapped) {
-                if (ctx->registered_frames[i].regptr) {
-                    nv_status = p_nvenc->nvEncUnregisterResource(ctx->nvencoder, ctx->registered_frames[i].regptr);
-                    if (nv_status != NV_ENC_SUCCESS)
-                        return nvenc_print_error(avctx, nv_status, "Failed unregistering unused input resource");
-                    ctx->registered_frames[i].ptr = NULL;
-                    ctx->registered_frames[i].regptr = NULL;
+        for (first_round = 1; first_round >= 0; first_round--) {
+            for (i = 0; i < ctx->nb_registered_frames; i++) {
+                if (!ctx->registered_frames[i].mapped) {
+                    if (ctx->registered_frames[i].regptr) {
+                        if (first_round)
+                            continue;
+                        nv_status = p_nvenc->nvEncUnregisterResource(ctx->nvencoder, ctx->registered_frames[i].regptr);
+                        if (nv_status != NV_ENC_SUCCESS)
+                            return nvenc_print_error(avctx, nv_status, "Failed unregistering unused input resource");
+                        ctx->registered_frames[i].ptr = NULL;
+                        ctx->registered_frames[i].regptr = NULL;
+                    }
+                    return i;
                 }
-                return i;
             }
         }
     } else {
@@ -1846,13 +1850,15 @@ static int process_output_surface(AVCodecContext *avctx, AVPacket *pkt, NvencSur
                 res = nvenc_print_error(avctx, nv_status, "Failed unmapping input resource");
                 goto error;
             }
+#if NVENC_UNREGISTER_IMMEDIATELY
             nv_status = p_nvenc->nvEncUnregisterResource(ctx->nvencoder, ctx->registered_frames[tmpoutsurf->reg_idx].regptr);
             if (nv_status != NV_ENC_SUCCESS) {
                 res = nvenc_print_error(avctx, nv_status, "Failed unregistering input resource");
                 goto error;
             }
             ctx->registered_frames[tmpoutsurf->reg_idx].ptr = NULL;
             ctx->registered_frames[tmpoutsurf->reg_idx].regptr = NULL;
+#endif
         } else if (ctx->registered_frames[tmpoutsurf->reg_idx].mapped < 0) {
             res = AVERROR_BUG;
             goto error;

libavcodec/nvenc.h

@@ -54,6 +54,19 @@ typedef void ID3D11Device;
 #define NVENC_HAVE_HEVC_BFRAME_REF_MODE
 #endif
 
+/*
+ * If this is set to 1 at compile time, nvenc will unregister every input frame
+ * the moment it's no longer used. By default it will hold onto the device handle
+ * until it runs out of internal space to buffer them, which increases frame throughput.
+ *
+ * This is here so a potential API user, who for some reason is unable to make sure each
+ * and every pointer is available until nvenc is closed, can easily produce a custom
+ * build restoring the more compatible but slower behaviour.
+ */
+#ifndef NVENC_UNREGISTER_IMMEDIATELY
+#define NVENC_UNREGISTER_IMMEDIATELY 0
+#endif
+
 typedef struct NvencSurface
 {
     NV_ENC_INPUT_PTR input_surface;