libav: try to tidy up the mess that is Makefile.ffmpeg

- fix broken libx265 build
- rename libffmpeg_static to ffmpeg_static
- rename libmfx option to qsv (which is what one really wants)
- update to ffmpeg 3.0
- drop support for non-ffmpeg libav libs
  (kinda needed for libswresample, but might force users to upgrade)
- drop vdpau config option (unused)
- add multiple options (libvpx, libvorbis, ...) to customize builds to one's needs
- add libfdk-aac support
- add libtheora support
- try to cleanup the libav section in Makefile
- libx264 now supports swscale and avfilter
- patch libmfx to support the correct path to libmfx(s|h)w64.so on ubuntu
  (other distro users might not be pleased with that one)

Makefile

@@ -62,62 +62,83 @@ CFLAGS  += -Wno-unused-value -Wno-tautological-constant-out-of-range-compare
 CFLAGS  += -Wno-parentheses-equality -Wno-incompatible-pointer-types
 endif
 
-ifeq ($(CONFIG_LIBFFMPEG_STATIC),yes)
-
-CFLAGS  += -I${BUILDDIR}/ffmpeg/build/ffmpeg/include
-LDFLAGS_FFDIR = ${BUILDDIR}/ffmpeg/build/ffmpeg/lib
-LDFLAGS += ${LDFLAGS_FFDIR}/libavresample.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libswresample.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libavfilter.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libswscale.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libavutil.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libavformat.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libavcodec.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libavutil.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libvorbisenc.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libvorbis.a
-LDFLAGS += ${LDFLAGS_FFDIR}/libogg.a
+
+# LIBAV ########################################################################
+
+ifeq ($(CONFIG_LIBAV),yes)
+
+FFMPEG_LIBS := \
+    libavfilter \
+    libswresample \
+    libavresample \
+    libswscale \
+    libavformat \
+    libavcodec \
+    libavutil
+
+# FFMEG_STATIC
+ifeq ($(CONFIG_FFMPEG_STATIC),yes)
+
+FFMPEG_PREFIX := $(BUILDDIR)/ffmpeg/build/ffmpeg
+FFMPEG_LIBDIR := $(FFMPEG_PREFIX)/lib
+FFMPEG_CONFIG := \
+    PKG_CONFIG_LIBDIR=$(FFMPEG_LIBDIR)/pkgconfig $(PKG_CONFIG) \
+    --define-variable=prefix=$(FFMPEG_PREFIX) --static
 
 ifeq ($(CONFIG_LIBX264_STATIC),yes)
-LDFLAGS += ${LDFLAGS_FFDIR}/libx264.a -ldl
-else
-LDFLAGS += -lx264 -ldl
+FFMPEG_DEPS += libx264
 endif
 
-ifeq ($(CONFIG_LIBX265),yes)
 ifeq ($(CONFIG_LIBX265_STATIC),yes)
-LDFLAGS += ${LDFLAGS_FFDIR}/libx265.a -lstdc++
-else
-LDFLAGS += -lx265
+FFMPEG_DEPS += libx265
 endif
+
+ifeq ($(CONFIG_LIBVPX_STATIC),yes)
+FFMPEG_DEPS += libvpx
 endif
 
-LDFLAGS += ${LDFLAGS_FFDIR}/libvpx.a
+ifeq ($(CONFIG_LIBOGG_STATIC),yes)
+FFMPEG_DEPS += libogg
+endif
 
-CONFIG_LIBMFX_VA_LIBS =
+ifeq ($(CONFIG_LIBTHEORA_STATIC),yes)
+FFMPEG_DEPS += libtheoraenc libtheoradec libtheora
+endif
 
-ifeq ($(CONFIG_LIBMFX),yes)
-CONFIG_LIBMFX_VA_LIBS += -lva
-ifeq ($(CONFIG_VA_DRM),yes)
-CONFIG_LIBMFX_VA_LIBS += -lva-drm
+ifeq ($(CONFIG_LIBVORBIS_STATIC),yes)
+FFMPEG_DEPS += libvorbisfile libvorbisenc libvorbis
 endif
-ifeq ($(CONFIG_VA_X11),yes)
-CONFIG_LIBMFX_VA_LIBS += -lva-x11
+
+ifeq ($(CONFIG_LIBFDKAAC_STATIC),yes)
+FFMPEG_DEPS += libfdk-aac
 endif
+
 ifeq ($(CONFIG_LIBMFX_STATIC),yes)
-LDFLAGS += ${LDFLAGS_FFDIR}/libmfx.a -lstdc++
-else
-LDFLAGS += -lmfx
+FFMPEG_DEPS += libmfx
 endif
-LDFLAGS += ${CONFIG_LIBMFX_VA_LIBS}
+
+LDFLAGS += $(foreach lib,$(FFMPEG_LIBS),$(FFMPEG_LIBDIR)/$(lib).a)
+LDFLAGS += $(foreach lib,$(FFMPEG_DEPS),$(FFMPEG_LIBDIR)/$(lib).a)
+
+else
+
+FFMPEG_CONFIG := $(PKG_CONFIG)
+
+endif # FFMEG_STATIC
+
+CFLAGS  += `$(FFMPEG_CONFIG) --cflags $(FFMPEG_LIBS)`
+LDFLAGS += `$(FFMPEG_CONFIG) --libs $(FFMPEG_LIBS)`
+
 endif
 
-endif # CONFIG_LIBFFMPEG_STATIC
+# LIBAV ########################################################################
+
 
 ifeq ($(CONFIG_HDHOMERUN_STATIC),yes)
 CFLAGS  += -I$(BUILDDIR)/hdhomerun
-LDFLAGS += $(BUILDDIR)/hdhomerun/libhdhomerun/libhdhomerun.a \
-           -Wl,-Bstatic -Wl,-Bdynamic
+LDFLAGS += \
+    $(BUILDDIR)/hdhomerun/libhdhomerun/libhdhomerun.a \
+    -L$(BUILDDIR)/hdhomerun/libhdhomerun
 endif
 
 vpath %.c $(ROOTDIR)
@@ -453,7 +474,7 @@ SRCS-CWC = \
 	src/descrambler/emm_reass.c
 SRCS-${CONFIG_CWC} += $(SRCS-CWC)
 I18N-C += $(SRCS-CWC)
-	
+
 # CAPMT
 SRCS-CAPMT = \
 	src/descrambler/capmt.c
@@ -541,7 +562,7 @@ OBJS       = $(SRCS:%.c=$(BUILDDIR)/%.o)
 OBJS_EXTRA = $(SRCS_EXTRA:%.c=$(BUILDDIR)/%.so)
 DEPS       = ${OBJS:%.o=%.d}
 
-ifeq ($(CONFIG_LIBFFMPEG_STATIC),yes)
+ifeq ($(CONFIG_FFMPEG_STATIC),yes)
 ALL-yes   += ${BUILDDIR}/libffmpeg_stamp
 endif
 ifeq ($(CONFIG_HDHOMERUN_STATIC),yes)
@@ -684,7 +705,7 @@ make_webui:
 
 # Static FFMPEG
 
-ifeq ($(CONFIG_LIBFFMPEG_STATIC),yes)
+ifeq ($(CONFIG_FFMPEG_STATIC),yes)
 src/libav.h ${SRCS-LIBAV} ${DEPS-LIBAV}: ${BUILDDIR}/libffmpeg_stamp
 endif