Clarify NVENC live view startup failures

server/pkg/gst/gst.c

@@ -1,5 +1,85 @@
 #include "gst.h"
 
+#include <dlfcn.h>
+
+#define CUDA_ERROR_NO_DEVICE 100
+
+typedef int CUdevice;
+typedef void *CUcontext;
+typedef int CUresult;
+
+typedef CUresult (*CuInit)(unsigned int flags);
+typedef CUresult (*CuDeviceGetCount)(int *count);
+typedef CUresult (*CuDeviceGet)(CUdevice *device, int ordinal);
+typedef CUresult (*CuCtxCreate)(CUcontext *context, unsigned int flags, CUdevice device);
+typedef CUresult (*CuCtxDestroy)(CUcontext context);
+typedef CUresult (*CuGetErrorName)(CUresult error, const char **name);
+
+static int cuda_probe_result(void *library, CUresult code, const char *stage,
+    CuGetErrorName getErrorName, char **resultStage, char **errorName) {
+  const char *name = NULL;
+  if (getErrorName(code, &name) != 0 || name == NULL) {
+    name = "CUDA_ERROR_UNKNOWN";
+  }
+
+  *resultStage = g_strdup(stage);
+  *errorName = g_strdup(name);
+  dlclose(library);
+  return code;
+}
+
+int gstreamer_cuda_context_probe(char **stage, char **errorName) {
+  void *library = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE);
+  if (library == NULL) {
+    *stage = g_strdup("loading the CUDA driver");
+    *errorName = g_strdup("CUDA_DRIVER_LIBRARY_UNAVAILABLE");
+    return -1;
+  }
+
+  CuInit cuInit = (CuInit)dlsym(library, "cuInit");
+  CuDeviceGetCount cuDeviceGetCount = (CuDeviceGetCount)dlsym(library, "cuDeviceGetCount");
+  CuDeviceGet cuDeviceGet = (CuDeviceGet)dlsym(library, "cuDeviceGet");
+  CuCtxCreate cuCtxCreate = (CuCtxCreate)dlsym(library, "cuCtxCreate_v2");
+  CuCtxDestroy cuCtxDestroy = (CuCtxDestroy)dlsym(library, "cuCtxDestroy_v2");
+  CuGetErrorName cuGetErrorName = (CuGetErrorName)dlsym(library, "cuGetErrorName");
+  if (cuInit == NULL || cuDeviceGetCount == NULL || cuDeviceGet == NULL ||
+      cuCtxCreate == NULL || cuCtxDestroy == NULL || cuGetErrorName == NULL) {
+    *stage = g_strdup("resolving CUDA driver symbols");
+    *errorName = g_strdup("CUDA_DRIVER_SYMBOL_UNAVAILABLE");
+    dlclose(library);
+    return -2;
+  }
+
+  CUresult result = cuInit(0);
+  if (result != 0) {
+    return cuda_probe_result(library, result, "initializing CUDA", cuGetErrorName, stage, errorName);
+  }
+
+  int deviceCount = 0;
+  result = cuDeviceGetCount(&deviceCount);
+  if (result != 0) {
+    return cuda_probe_result(library, result, "querying CUDA devices", cuGetErrorName, stage, errorName);
+  }
+  if (deviceCount == 0) {
+    return cuda_probe_result(library, CUDA_ERROR_NO_DEVICE, "querying CUDA devices", cuGetErrorName, stage, errorName);
+  }
+
+  CUdevice device;
+  result = cuDeviceGet(&device, 0);
+  if (result != 0) {
+    return cuda_probe_result(library, result, "selecting a CUDA device", cuGetErrorName, stage, errorName);
+  }
+
+  CUcontext context;
+  result = cuCtxCreate(&context, 0, device);
+  if (result != 0) {
+    return cuda_probe_result(library, result, "creating a CUDA context", cuGetErrorName, stage, errorName);
+  }
+
+  cuCtxDestroy(context);
+  return cuda_probe_result(library, 0, "creating a CUDA context", cuGetErrorName, stage, errorName);
+}
+
 static void gstreamer_pipeline_log(GstPipelineCtx *ctx, char* level, const char* format, ...) {
   va_list argptr;
   va_start(argptr, format);

server/pkg/gst/gst.go

@@ -2,12 +2,14 @@ package gst
 
 /*
 #cgo pkg-config: gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0
+#cgo LDFLAGS: -ldl
 
 #include "gst.h"
 */
 import "C"
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -59,20 +61,25 @@ type pipeline struct {
 }
 
 func CreatePipeline(pipelineStr string) (Pipeline, error) {
+	return createPipeline(pipelineStr, probeCUDAContext)
+}
+
+func createPipeline(pipelineStr string, probe func() cudaProbeResult) (Pipeline, error) {
 	id := atomic.AddInt32(&pSerial, 1)
 
 	pipelineStrUnsafe := C.CString(pipelineStr)
 	defer C.free(unsafe.Pointer(pipelineStrUnsafe))
 
 	pipelinesLock.Lock()
-	defer pipelinesLock.Unlock()
 
 	var gstError *C.GError
 	ctx := C.gstreamer_pipeline_create(pipelineStrUnsafe, C.int(id), &gstError)
 
 	if gstError != nil {
+		pipelinesLock.Unlock()
 		defer C.g_error_free(gstError)
-		return nil, fmt.Errorf("(pipeline error) %s", C.GoString(gstError.message))
+		msg := annotatePipelineError(pipelineStr, C.GoString(gstError.message), probe)
+		return nil, fmt.Errorf("(pipeline error) %s", msg)
 	}
 
 	p := &pipeline{
@@ -87,9 +94,76 @@ func CreatePipeline(pipelineStr string) (Pipeline, error) {
 	}
 
 	pipelines[p.id] = p
+	pipelinesLock.Unlock()
 	return p, nil
 }
 
+const (
+	cudaDriverLibraryUnavailable = -1
+	cudaDriverSymbolUnavailable  = -2
+	cudaSuccess                  = 0
+	cudaErrorOutOfMemory         = 2
+	cudaErrorNoDevice            = 100
+)
+
+type cudaProbeResult struct {
+	code  int
+	stage string
+	name  string
+}
+
+func annotatePipelineError(pipelineStr, msg string, probe func() cudaProbeResult) string {
+	if !isMissingNVENCElementError(pipelineStr, msg) {
+		return msg
+	}
+
+	return fmt.Sprintf("%s (%s)", msg, nvencFailureDetail(probe()))
+}
+
+func isMissingNVENCElementError(pipelineStr, msg string) bool {
+	lowerMsg := strings.ToLower(msg)
+	return strings.Contains(pipelineStr, "nvh264enc") &&
+		strings.Contains(lowerMsg, "nvh264enc") &&
+		(strings.Contains(lowerMsg, "no element") || strings.Contains(lowerMsg, "no such element or plugin"))
+}
+
+func probeCUDAContext() cudaProbeResult {
+	var stage, name *C.char
+	code := int(C.gstreamer_cuda_context_probe(&stage, &name))
+
+	if stage != nil {
+		defer C.g_free(C.gpointer(stage))
+	}
+	if name != nil {
+		defer C.g_free(C.gpointer(name))
+	}
+
+	return cudaProbeResult{
+		code:  code,
+		stage: C.GoString(stage),
+		name:  C.GoString(name),
+	}
+}
+
+func nvencFailureDetail(probe cudaProbeResult) string {
+	const prefix = "live view could not initialize NVENC/CUDA"
+
+	switch probe.code {
+	case cudaDriverLibraryUnavailable:
+		return prefix + ": the CUDA driver library is unavailable"
+	case cudaDriverSymbolUnavailable:
+		return prefix + ": required CUDA driver symbols are unavailable"
+	case cudaSuccess:
+		return prefix + ": the CUDA context probe succeeded; possible causes are failed GStreamer nvcodec registration, an unavailable NVIDIA encode library, a driver or capability mismatch, or exhausted NVENC sessions"
+	case cudaErrorOutOfMemory:
+		return fmt.Sprintf("%s: CUDA reported %s (%d) while %s. GPU memory is exhausted; reduce browser resolution or stop replay/browser GPU load, then restart Neko before retrying live view", prefix, probe.name, probe.code, probe.stage)
+	case cudaErrorNoDevice:
+		return fmt.Sprintf("%s: CUDA reported %s (%d) while %s; no CUDA-capable GPU is available to Neko", prefix, probe.name, probe.code, probe.stage)
+	default:
+		return fmt.Sprintf("%s: CUDA reported %s (%d) while %s", prefix, probe.name, probe.code, probe.stage)
+	}
+}
+
 func (p *pipeline) Src() string {
 	return p.src
 }

server/pkg/gst/gst.h

@@ -38,3 +38,4 @@ gboolean gstreamer_pipeline_set_prop_int(GstPipelineCtx *ctx, char *binName, cha
 gboolean gstreamer_pipeline_set_caps_framerate(GstPipelineCtx *ctx, const gchar* binName, gint numerator, gint denominator);
 gboolean gstreamer_pipeline_set_caps_resolution(GstPipelineCtx *ctx, const gchar* binName, gint width, gint height);
 gboolean gstreamer_pipeline_emit_video_keyframe(GstPipelineCtx *ctx);
+int gstreamer_cuda_context_probe(char **stage, char **errorName);

server/pkg/gst/gst_test.go

@@ -0,0 +1,149 @@
+package gst
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestIsMissingNVENCElementError(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		pipelineStr string
+		msg         string
+		want        bool
+	}{
+		{
+			name:        "missing nvh264enc in gpu pipeline",
+			pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink",
+			msg:         `no element "nvh264enc"`,
+			want:        true,
+		},
+		{
+			name:        "alternate plugin wording",
+			pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink",
+			msg:         "No such element or plugin 'nvh264enc'",
+			want:        true,
+		},
+		{
+			name:        "unrelated encoder",
+			pipelineStr: "ximagesrc ! x264enc name=encoder ! appsink name=appsink",
+			msg:         `no element "x264enc"`,
+		},
+		{
+			name:        "other nvh264enc error",
+			pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink",
+			msg:         "could not link cudaupload to nvh264enc",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			if got := isMissingNVENCElementError(tt.pipelineStr, tt.msg); got != tt.want {
+				t.Fatalf("isMissingNVENCElementError(%q, %q) = %v, want %v", tt.pipelineStr, tt.msg, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCreatePipelineReleasesLockBeforeSlowCUDAProbe(t *testing.T) {
+	probeStarted := make(chan struct{})
+	releaseProbe := make(chan struct{})
+	pipelineDone := make(chan error, 1)
+
+	go func() {
+		_, err := createPipeline("nvh264enc_missing", func() cudaProbeResult {
+			close(probeStarted)
+			<-releaseProbe
+			return cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"}
+		})
+		pipelineDone <- err
+	}()
+
+	select {
+	case <-probeStarted:
+	case err := <-pipelineDone:
+		t.Fatalf("pipeline creation returned before running probe: %v", err)
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for CUDA probe")
+	}
+
+	lockAcquired := make(chan struct{})
+	go func() {
+		pipelinesLock.Lock()
+		pipelinesLock.Unlock()
+		close(lockAcquired)
+	}()
+
+	select {
+	case <-lockAcquired:
+		close(releaseProbe)
+	case <-time.After(time.Second):
+		close(releaseProbe)
+		<-pipelineDone
+		<-lockAcquired
+		t.Fatal("pipeline lock remained held during CUDA probe")
+	}
+
+	if err := <-pipelineDone; err == nil {
+		t.Fatal("createPipeline() error = nil, want missing element error")
+	}
+}
+
+func TestNVENCFailureDetail(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name  string
+		probe cudaProbeResult
+		want  string
+	}{
+		{
+			name:  "driver library unavailable",
+			probe: cudaProbeResult{code: cudaDriverLibraryUnavailable},
+			want:  "CUDA driver library is unavailable",
+		},
+		{
+			name:  "driver symbols unavailable",
+			probe: cudaProbeResult{code: cudaDriverSymbolUnavailable},
+			want:  "required CUDA driver symbols are unavailable",
+		},
+		{
+			name:  "cuda succeeds",
+			probe: cudaProbeResult{code: cudaSuccess},
+			want:  "CUDA context probe succeeded",
+		},
+		{
+			name:  "out of memory",
+			probe: cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"},
+			want:  "CUDA_ERROR_OUT_OF_MEMORY (2) while creating a CUDA context. GPU memory is exhausted",
+		},
+		{
+			name:  "no device",
+			probe: cudaProbeResult{code: cudaErrorNoDevice, stage: "querying CUDA devices", name: "CUDA_ERROR_NO_DEVICE"},
+			want:  "CUDA_ERROR_NO_DEVICE (100) while querying CUDA devices; no CUDA-capable GPU is available",
+		},
+		{
+			name:  "other cuda failure",
+			probe: cudaProbeResult{code: 999, stage: "initializing CUDA", name: "CUDA_ERROR_UNKNOWN"},
+			want:  "CUDA_ERROR_UNKNOWN (999) while initializing CUDA",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := nvencFailureDetail(tt.probe)
+			if !strings.Contains(got, tt.want) {
+				t.Fatalf("nvencFailureDetail() = %q, want substring %q", got, tt.want)
+			}
+		})
+	}
+}